mirror of
https://github.com/OpenBMB/MiniCPM-V.git
synced 2026-02-05 18:29:18 +08:00
Update to MiniCPM-o 2.6
This commit is contained in:
yiranyyu
7
web_demos/minicpm-o_2.6/web_server/src/App.vue
Normal file
7
web_demos/minicpm-o_2.6/web_server/src/App.vue
Normal file
@@ -0,0 +1,7 @@
|
||||
<template>
|
||||
<RouterView />
|
||||
</template>
|
||||
|
||||
<script setup></script>
|
||||
|
||||
<style lang="less" scoped></style>
|
||||
21
web_demos/minicpm-o_2.6/web_server/src/apis/index.js
Normal file
21
web_demos/minicpm-o_2.6/web_server/src/apis/index.js
Normal file
@@ -0,0 +1,21 @@
|
||||
// 定时发送消息
|
||||
export const sendMessage = data => {
|
||||
return useHttp.post('/api/v1/stream', data);
|
||||
};
|
||||
// 跳过当前
|
||||
export const stopMessage = () => {
|
||||
return useHttp.post('/api/v1/stop');
|
||||
};
|
||||
// 上传音色文件
|
||||
export const uploadFile = data => {
|
||||
return useHttp.post('/api/v1/upload_audio', data);
|
||||
};
|
||||
// 反馈
|
||||
export const feedback = data => {
|
||||
return useHttp.post('/api/v1/feedback', data);
|
||||
};
|
||||
// 上传配置
|
||||
export const uploadConfig = data => {
|
||||
return useHttp.post('/api/v1/init_options', data);
|
||||
// return useHttp.post('/api/v1/upload_audio', data);
|
||||
};
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 221 B |
BIN
web_demos/minicpm-o_2.6/web_server/src/assets/images/cai.png
Normal file
BIN
web_demos/minicpm-o_2.6/web_server/src/assets/images/cai.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 284 B |
Binary file not shown.
|
After Width: | Height: | Size: 1.5 KiB |
BIN
web_demos/minicpm-o_2.6/web_server/src/assets/images/logo.png
Normal file
BIN
web_demos/minicpm-o_2.6/web_server/src/assets/images/logo.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 6.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
BIN
web_demos/minicpm-o_2.6/web_server/src/assets/images/voice.png
Normal file
BIN
web_demos/minicpm-o_2.6/web_server/src/assets/images/voice.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 2.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 391 B |
BIN
web_demos/minicpm-o_2.6/web_server/src/assets/images/zan.png
Normal file
BIN
web_demos/minicpm-o_2.6/web_server/src/assets/images/zan.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 279 B |
@@ -0,0 +1 @@
|
||||
<svg data-v-d2e47025="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1024 1024"><path fill="currentColor" d="M600.704 64a32 32 0 0 1 30.464 22.208l35.2 109.376c14.784 7.232 28.928 15.36 42.432 24.512l112.384-24.192a32 32 0 0 1 34.432 15.36L944.32 364.8a32 32 0 0 1-4.032 37.504l-77.12 85.12a357.12 357.12 0 0 1 0 49.024l77.12 85.248a32 32 0 0 1 4.032 37.504l-88.704 153.6a32 32 0 0 1-34.432 15.296L708.8 803.904c-13.44 9.088-27.648 17.28-42.368 24.512l-35.264 109.376A32 32 0 0 1 600.704 960H423.296a32 32 0 0 1-30.464-22.208L357.696 828.48a351.616 351.616 0 0 1-42.56-24.64l-112.32 24.256a32 32 0 0 1-34.432-15.36L79.68 659.2a32 32 0 0 1 4.032-37.504l77.12-85.248a357.12 357.12 0 0 1 0-48.896l-77.12-85.248A32 32 0 0 1 79.68 364.8l88.704-153.6a32 32 0 0 1 34.432-15.296l112.32 24.256c13.568-9.152 27.776-17.408 42.56-24.64l35.2-109.312A32 32 0 0 1 423.232 64H600.64zm-23.424 64H446.72l-36.352 113.088-24.512 11.968a294.113 294.113 0 0 0-34.816 20.096l-22.656 15.36-116.224-25.088-65.28 113.152 79.68 88.192-1.92 27.136a293.12 293.12 0 0 0 0 40.192l1.92 27.136-79.808 88.192 65.344 113.152 116.224-25.024 22.656 15.296a294.113 294.113 0 0 0 34.816 20.096l24.512 11.968L446.72 896h130.688l36.48-113.152 24.448-11.904a288.282 288.282 0 0 0 34.752-20.096l22.592-15.296 116.288 25.024 65.28-113.152-79.744-88.192 1.92-27.136a293.12 293.12 0 0 0 0-40.256l-1.92-27.136 79.808-88.128-65.344-113.152-116.288 24.96-22.592-15.232a287.616 287.616 0 0 0-34.752-20.096l-24.448-11.904L577.344 128zM512 320a192 192 0 1 1 0 384 192 192 0 0 1 0-384m0 64a128 128 0 1 0 0 256 128 128 0 0 0 0-256"></path></svg>
|
||||
|
After Width: | Height: | Size: 1.6 KiB |
@@ -0,0 +1 @@
|
||||
<svg data-v-d2e47025="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1024 1024"><path fill="currentColor" d="M832 384H576V128H192v768h640zm-26.496-64L640 154.496V320zM160 64h480l256 256v608a32 32 0 0 1-32 32H160a32 32 0 0 1-32-32V96a32 32 0 0 1 32-32m160 448h384v64H320zm0-192h160v64H320zm0 384h384v64H320z"></path></svg>
|
||||
|
After Width: | Height: | Size: 324 B |
@@ -0,0 +1,5 @@
|
||||
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<g id="Icon/Utility Icon/line/error">
|
||||
<path id="Union" fill-rule="evenodd" clip-rule="evenodd" d="M9.99997 20C4.48608 20 0 15.5139 0 10C0 4.48607 4.48606 0 9.99997 0C15.5139 0 19.9999 4.48609 19.9999 10C19.9999 15.5139 15.5139 20 9.99997 20ZM9.99997 1.875C5.52001 1.875 1.875 5.52002 1.875 10C1.875 14.48 5.52001 18.125 9.99997 18.125C14.4799 18.125 18.125 14.48 18.125 10C18.125 5.52002 14.4799 1.875 9.99997 1.875ZM13.7878 7.53784L11.3257 9.99999L13.7878 12.4621C14.154 12.8283 14.154 13.4216 13.7878 13.7878C13.6047 13.9709 13.3655 14.0625 13.125 14.0625C12.8845 14.0625 12.6452 13.9709 12.4621 13.7878L9.99998 11.3257L7.53784 13.7878C7.35473 13.9709 7.11548 14.0625 6.875 14.0625C6.63451 14.0625 6.39526 13.9709 6.21216 13.7878C5.84595 13.4216 5.84595 12.8283 6.21216 12.4621L8.6743 9.99999L6.21216 7.53784C5.84595 7.17163 5.84595 6.57837 6.21216 6.21216C6.57836 5.84595 7.17163 5.84595 7.53784 6.21216L10 8.67431L12.4621 6.21216C12.8283 5.84595 13.4216 5.84595 13.7878 6.21216C14.154 6.57837 14.154 7.17163 13.7878 7.53784Z" fill="#E72B00"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.1 KiB |
@@ -0,0 +1,29 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg viewBox="0 0 2199 258" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<title>编组 5</title>
|
||||
<defs>
|
||||
<linearGradient x1="45.9111958%" y1="57.6904311%" x2="4.78458419e-14%" y2="70.534914%" id="linearGradient-1">
|
||||
<stop stop-color="#373ED8" offset="0%"></stop>
|
||||
<stop stop-color="#497DFF" offset="100%"></stop>
|
||||
</linearGradient>
|
||||
<path d="M1812.80909,215.823442 L1812.80909,252.015442 L1952.00909,252.015442 L1952.00909,211.995442 L1870.22909,211.995442 L1930.08509,134.391442 C1937.27682,125.111446 1942.72882,116.063446 1946.44109,107.247442 C1950.15309,98.4314389 1952.00909,88.8034425 1952.00909,78.3634425 L1952.00909,72.4474425 C1952.00909,60.6154425 1949.16709,49.8274425 1943.48309,40.0834425 C1937.79935,30.3394425 1929.67935,22.5674425 1919.12309,16.7674425 C1908.56709,10.9674354 1896.32909,8.06744248 1882.40909,8.06744248 C1868.02509,8.06744248 1855.49709,10.8514425 1844.82509,16.4194425 C1834.15309,21.9874425 1825.97509,29.6434425 1820.29109,39.3874425 C1814.6071,49.1314425 1811.76509,60.2674425 1811.76509,72.7954425 L1811.76509,81.1474425 L1855.96109,81.1474425 L1855.96109,75.5794425 C1855.96109,66.5314425 1858.16509,59.4554496 1862.57309,54.3514425 C1866.98109,49.2474354 1873.01309,46.6954425 1880.66909,46.6954425 C1888.32509,46.6954425 1894.41509,49.1314425 1898.93909,54.0034425 C1903.46309,58.8754425 1905.72509,65.4874425 1905.72509,73.8394425 L1905.72509,78.7114425 C1905.72509,89.3834389 1901.31709,100.635442 1892.50109,112.467442 L1812.80909,215.823442 Z M1976.89309,202.599442 L1976.89309,252.015442 L2025.26509,252.015442 L2025.26509,202.599442 L1976.89309,202.599442 Z M2069.81109,237.051442 C2082.91909,249.579442 2101.07309,255.843442 2124.27309,255.843442 C2146.54509,255.843442 2164.46709,249.463444 2178.03909,236.703442 C2191.61109,223.943441 2198.39709,206.195446 2198.39709,183.459442 L2198.39709,172.323442 C2198.39709,151.675439 2192.77109,135.377446 2181.51909,123.429442 C2170.26709,111.481439 2155.24509,105.507442 2136.45309,105.507442 C2129.95709,105.507442 2124.15709,106.667446 2119.05309,108.987442 L2168.81709,11.8954425 L2120.79309,11.8954425 L2065.80909,118.731442 C2060.70509,128.939446 2056.81909,138.335446 2054.15109,146.919442 C2051.48309,155.503439 2050.14909,164.551444 2050.14909,174.063442 L2050.14909,184.851442 C2050.14909,207.123442 2056.70309,224.523442 2069.81109,237.051442 Z M2145.15309,208.863442 C2140.04909,214.431442 2133.08909,217.215442 2124.27309,217.215442 C2115.45709,217.215442 2108.55509,214.431442 2103.56709,208.863442 C2098.57909,203.295442 2096.08509,195.639442 2096.08509,185.895442 L2096.08509,174.411442 C2096.08509,164.667448 2098.57909,157.06945 2103.56709,151.617442 C2108.55507,146.165446 2115.45707,143.439442 2124.27309,143.439442 C2133.08909,143.439442 2140.04909,146.165446 2145.15309,151.617442 C2150.25709,157.069439 2152.80909,164.783441 2152.80909,174.759442 L2152.80909,185.547442 C2152.80909,195.523444 2150.25709,203.295442 2145.15309,208.863442 Z" id="path-2"></path>
|
||||
</defs>
|
||||
<g id="页面-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
|
||||
<g id="画板备份-14" transform="translate(-1928, -1764)" fill-rule="nonzero">
|
||||
<g id="编组-5" transform="translate(1928, 1764.9846)">
|
||||
<path d="M760.177408,6.08426104 C780.959767,6.08426104 798.639412,11.1994393 813.310847,21.4099653 L814.826937,22.4868416 C827.871266,31.9421726 838.44267,44.5805385 846.551989,60.4441981 L846.854209,61.0497535 L805.164914,79.6080391 L804.700192,78.6693484 C800.249247,69.9367001 794.263119,63.1340103 786.753168,58.3199387 C778.149995,52.8050845 768.233984,50.0506369 757.083763,50.0506369 C744.833865,50.0506369 733.831738,53.5163071 724.177049,60.4281868 C714.581392,67.2978043 707.134706,76.9187061 701.846243,89.2224762 C696.604371,101.417853 693.991732,115.174193 693.991732,130.467076 C693.991732,146.161685 696.600155,160.26846 701.833896,172.765351 C707.114285,185.373628 714.549996,195.251726 724.136357,202.332561 C733.797042,209.468294 744.814803,213.049067 757.083763,213.049067 C767.641468,213.049067 777.212416,210.227573 785.712813,204.59744 L787.029919,203.697896 C793.997044,198.793482 800.045118,192.1801 805.176059,183.885785 L805.493229,183.359111 L847.470223,202.046016 L847.190902,202.601562 C838.534575,219.439902 826.824502,232.527091 812.037303,241.920218 C796.196991,251.982303 778.007922,257.015442 757.393128,257.015442 C735.537395,257.015442 716.157915,251.671101 699.179392,240.984619 C682.183395,230.287139 668.940168,215.394755 659.416171,196.246509 C649.861213,177.036014 645.075526,155.122604 645.075526,130.467076 C645.075526,106.236416 649.907085,84.6957154 659.55475,65.8023698 C669.170256,46.9720039 682.655972,32.3375052 700.053275,21.8391328 C717.453143,11.3392121 737.472005,6.08426104 760.177408,6.08426104 Z M472.804069,70.4320631 C490.215347,70.4320631 503.551514,75.9687387 513.08592,87.0440588 L513.922858,88.0433234 C522.993334,99.1752071 527.579887,114.927363 527.579887,135.416907 L527.579887,252.681061 L482.065262,252.681061 L482.066689,147.48212 C482.066689,137.230753 479.444996,128.966722 474.122797,122.834623 C468.710698,116.598944 461.272898,113.470346 452.076652,113.470346 C441.497963,113.470346 432.82714,116.858283 426.285393,123.629565 L425.517793,124.451905 C419.503248,131.122279 416.518055,139.98993 416.518055,150.885129 L416.517248,252.681061 L371.003248,252.681061 L371.003248,74.7611551 L416.517248,74.7611551 L416.518055,98.8935909 L423.358335,98.8935909 L424.345965,97.2602023 C429.414322,88.8779197 436.064565,82.3247602 444.331449,77.5591448 C452.566403,72.8119363 462.037327,70.4320631 472.804069,70.4320631 Z M605.481416,74.7611551 L605.481416,252.681061 L559.9708,252.681061 L559.9708,74.7611551 L605.481416,74.7611551 Z M335.78549,74.7611551 L335.78549,252.681061 L290.27149,252.681061 L290.27149,74.7611551 L335.78549,74.7611551 Z M0,10.4147082 L43.0533273,10.4147082 L122.24903,130.758127 L127.754059,130.758127 L206.947055,10.4147082 L250.000382,10.4147082 L250.000382,252.681061 L204.178374,252.681061 L204.180526,104.498777 L197.139835,104.498777 L143.308009,184.313596 L106.66868,184.313596 L52.5323692,105.736235 L45.5131981,105.736235 L45.5106162,252.681061 L0,252.681061 L0,10.4147082 Z M961.869908,10.4147082 C981.192886,10.4147082 997.923497,13.7715036 1012.08946,20.4554447 C1026.1438,27.0867173 1036.87643,36.5393059 1044.3624,48.8517555 C1051.8649,61.1913978 1055.62564,75.6900619 1055.62564,92.415251 C1055.62564,109.151297 1051.96203,123.607707 1044.65738,135.847946 C1037.36788,148.062782 1026.98507,157.46144 1013.43892,164.086199 C999.800233,170.756213 983.652345,174.105774 964.963553,174.105774 L922.598939,174.105774 L922.596926,252.681061 L875.228112,252.681061 L875.228112,10.4147082 L961.869908,10.4147082 Z M953.826433,52.8349168 L922.598939,52.8349168 L922.598939,131.686221 L953.826433,131.686221 C969.953126,131.686221 982.80491,128.309905 992.310812,121.456813 C1002.06659,114.423581 1007.0188,104.634314 1007.0188,92.415251 C1007.0188,80.2034681 1002.0737,70.3707714 992.331646,63.2341476 C982.822362,56.2680445 969.961757,52.8349168 953.826433,52.8349168 Z M335.78549,0 L335.78549,45.5106162 L290.27149,45.5106162 L290.27149,0 L335.78549,0 Z M605.119253,0 L605.119253,45.5106162 L559.605252,45.5106162 L559.605252,0 L605.119253,0 Z" id="形状" fill="#111111"></path>
|
||||
<g id="M-V" transform="translate(1084.9431, 11.7574)" fill="#000111">
|
||||
<polygon id="路径" points="44.394 239.184 0 239.184 0 0 41.676 0 119.894 123.216 121.706 123.216 200.226 0 241.902 0 241.902 239.184 197.508 239.184 197.508 85.466 195.696 85.466 137.41 176.368 104.492 176.368 46.206 86.372 44.394 86.372"></polygon>
|
||||
<polygon id="路径" points="274.216 96.942 374.48 96.942 374.48 138.014 274.216 138.014"></polygon>
|
||||
</g>
|
||||
<g id="o" transform="translate(1501.3431, 42.9174)" fill="#000111">
|
||||
<path d="M95.4,213.12 C75.96,213.12 59.04,208.8 44.64,200.16 C30.24,191.52 19.2,179.16 11.52,163.08 C3.84,147 0,128.16 0,106.56 C0,84.96 3.84,66.12 11.52,50.04 C19.2,33.96 30.24,21.6 44.64,12.96 C59.04,4.32 75.96,0 95.4,0 C114.84,0 131.76,4.32 146.16,12.96 C160.56,21.6 171.66,33.96 179.46,50.04 C187.26,66.12 191.16,84.96 191.16,106.56 C191.16,128.16 187.26,147 179.46,163.08 C171.66,179.16 160.56,191.52 146.16,200.16 C131.76,208.8 114.84,213.12 95.4,213.12 Z M95.4,169.92 C110.52,169.92 122.46,164.22 131.22,152.82 C139.98,141.42 144.36,126 144.36,106.56 C144.36,86.88 139.98,71.4 131.22,60.12 C122.46,48.84 110.52,43.2 95.4,43.2 C80.52,43.2 68.7,48.84 59.94,60.12 C51.18,71.4 46.8,86.88 46.8,106.56 C46.8,126 51.18,141.42 59.94,152.82 C68.7,164.22 80.52,169.92 95.4,169.92 Z" id="形状"></path>
|
||||
</g>
|
||||
<g id="形状结合">
|
||||
<use fill="#000111" xlink:href="#path-2"></use>
|
||||
<use fill="url(#linearGradient-1)" xlink:href="#path-2"></use>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 8.9 KiB |
@@ -0,0 +1,5 @@
|
||||
<svg width="18" height="18" viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<g id="Pause">
|
||||
<path id="Vector" fill-rule="evenodd" clip-rule="evenodd" d="M4.875 2.2522H7.125C7.5375 2.2522 7.875 2.5897 7.875 3.0022V15.0022C7.875 15.4147 7.5375 15.7522 7.125 15.7522H4.875C4.4625 15.7522 4.125 15.4147 4.125 15.0022V3.0022C4.125 2.5897 4.4625 2.2522 4.875 2.2522ZM10.875 2.2522H13.125C13.5375 2.2522 13.875 2.5897 13.875 3.0022V15.0022C13.875 15.4147 13.5375 15.7522 13.125 15.7522H10.875C10.4625 15.7522 10.125 15.4147 10.125 15.0022V3.0022C10.125 2.5897 10.4625 2.2522 10.875 2.2522Z" fill="currentColor" />
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 638 B |
@@ -0,0 +1,10 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none">
|
||||
<g clip-path="url(#clip0_7781_19663)">
|
||||
<path d="M21.7786 18.4946C22.7599 18.6754 23.7615 17.9845 23.7955 16.9048C23.827 15.9053 23.7672 14.6519 23.4533 13.4613C23.141 12.2768 22.5546 11.0725 21.4695 10.2892C20.3647 9.49176 18.7205 8.97497 17.1207 8.64947C15.4984 8.31938 13.8179 8.16607 12.5642 8.15054C10.9332 8.13034 8.67094 8.26243 6.60622 8.68941C5.57392 8.90289 4.56701 9.19489 3.70489 9.59193C2.85192 9.98474 2.07652 10.5096 1.58739 11.2247C0.257894 13.1683 0.172116 15.4886 0.325588 16.9453C0.436943 18.0022 1.45742 18.5535 2.36025 18.353C3.07081 18.1951 3.71743 18.0593 4.36845 17.9225C5.30139 17.7265 6.24339 17.5286 7.3955 17.2614C7.46587 17.2451 7.53161 17.2194 7.59169 17.1859C7.85982 17.0768 8.05173 16.8168 8.05917 16.509C8.09666 14.957 8.40578 14.0228 8.95698 13.4586C9.50108 12.9017 10.4369 12.5484 12.1227 12.5476C13.8976 12.5468 14.8691 12.862 15.4225 13.3997C15.9698 13.9314 16.2828 14.8523 16.2836 16.5335C16.2836 16.5634 16.2854 16.5928 16.2888 16.6217C16.279 16.6521 16.2711 16.6836 16.2651 16.7159C16.19 17.1233 16.4594 17.5144 16.8668 17.5894L21.7786 18.4946Z" fill="currentColor" />
|
||||
</g>
|
||||
<defs>
|
||||
<clipPath id="clip0_7781_19663">
|
||||
<rect width="24" height="24" fill="white"/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.3 KiB |
@@ -0,0 +1 @@
|
||||
<svg t="1736675176012" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4244" xmlns:xlink="http://www.w3.org/1999/xlink"><path d="M512 106.667A405.333 405.333 0 1 1 106.667 512 405.333 405.333 0 0 1 512 106.667m0-64A469.333 469.333 0 1 0 981.333 512 469.333 469.333 0 0 0 512 42.667z" p-id="4245"></path><path d="M501.333 664.533a32 32 0 1 0 32 32 32 32 0 0 0-32-32z m-0.426-27.093a32 32 0 0 1-32-32c0-80.213 50.56-111.787 91.306-136.96 32-19.84 51.84-33.28 59.094-60.16a85.333 85.333 0 0 0-12.587-69.547 91.52 91.52 0 0 0-76.8-29.226 123.52 123.52 0 0 0-92.16 29.866 82.56 82.56 0 0 0-21.333 52.907 32 32 0 1 1-64 2.56 144 144 0 0 1 39.466-99.84c31.574-32.853 78.08-49.493 138.24-49.493 70.827 0 108.587 29.44 128 54.186a149.333 149.333 0 0 1 23.894 125.014c-14.08 52.693-54.614 77.866-87.04 98.133-40.32 24.747-61.654 39.68-61.654 82.56a32 32 0 0 1-32.426 32z" p-id="4246"></path></svg>
|
||||
|
After Width: | Height: | Size: 931 B |
@@ -0,0 +1,3 @@
|
||||
<svg data-v-d2e47025="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1024 1024">
|
||||
<path fill="currentColor" d="M771.776 794.88A384 384 0 0 1 128 512h64a320 320 0 0 0 555.712 216.448H654.72a32 32 0 1 1 0-64h149.056a32 32 0 0 1 32 32v148.928a32 32 0 1 1-64 0v-50.56zM276.288 295.616h92.992a32 32 0 0 1 0 64H220.16a32 32 0 0 1-32-32V178.56a32 32 0 0 1 64 0v50.56A384 384 0 0 1 896.128 512h-64a320 320 0 0 0-555.776-216.384z"></path>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 442 B |
@@ -0,0 +1,7 @@
|
||||
<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<g id="下载">
|
||||
<rect width="24" height="24" rx="7" fill="#EAEFFF"/>
|
||||
<path id="Vector" d="M12.2816 16.1003C11.9134 16.1003 11.615 15.8019 11.615 15.4337V6.2513L8.28168 9.50983C8.11137 9.6765 7.86502 9.73978 7.63559 9.67571C7.40617 9.61165 7.22831 9.42989 7.16894 9.1989C7.10956 8.96817 7.17805 8.72313 7.34836 8.55646L11.8163 4.18987C12.0785 3.93363 12.4985 3.93727 12.7563 4.19794L17.088 8.56323C17.3399 8.82599 17.3344 9.24187 17.0763 9.49811C16.818 9.7541 16.4018 9.75592 16.1414 9.50202L12.9483 6.28489V15.4337C12.9483 15.8019 12.6498 16.1003 12.2816 16.1003Z" fill="#424EC5"/>
|
||||
<path id="Vector_2" d="M4.66666 13.6001C5.03488 13.6001 5.33331 13.8985 5.33331 14.2668V17.4667C5.33331 17.8349 5.63174 18.1334 5.99997 18.1334H17.9998C18.368 18.1334 18.6664 17.8349 18.6664 17.4667V14.2668C18.6664 13.8985 18.9648 13.6001 19.3331 13.6001C19.7013 13.6001 19.9997 13.8985 19.9997 14.2668V17.4667C19.9997 18.5714 19.1044 19.4667 17.9998 19.4667H5.99997C4.8953 19.4667 4 18.5714 4 17.4667V14.2668C4 13.8985 4.29843 13.6001 4.66666 13.6001Z" fill="#424EC5"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.2 KiB |
41
web_demos/minicpm-o_2.6/web_server/src/assets/svg/voice.svg
Normal file
41
web_demos/minicpm-o_2.6/web_server/src/assets/svg/voice.svg
Normal file
@@ -0,0 +1,41 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="195" height="45" viewBox="0 0 195 45" fill="none">
|
||||
<rect x="16" y="18.2257" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="11" y="18.2257" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="6" y="18.2257" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="0.907227" y="18.2257" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="71" y="18" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="91" y="18" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="111" y="18" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="131" y="18" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="66" y="18" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="86" y="18" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="106" y="18" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="126" y="18" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="61" y="18" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="81" y="18" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="101" y="18" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="121" y="18" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="56" y="18" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="76" y="18" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="96" y="18" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="116" y="18" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="21" y="13.3407" width="3.14815" height="18.3186" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect width="3.14815" height="18.3186" rx="1.57407" transform="matrix(-1 0 0 1 54 13.3849)" fill="#F3F3F3"/>
|
||||
<rect x="26" y="8.45581" width="3.14815" height="28.0885" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect width="3.14815" height="28.0885" rx="1.57407" transform="matrix(-1 0 0 1 49 8.5)" fill="#F3F3F3"/>
|
||||
<rect x="31" y="9.9823" width="3.14815" height="25.0354" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect width="3.14815" height="25.0354" rx="1.57407" transform="matrix(-1 0 0 1 44 10.0265)" fill="#F3F3F3"/>
|
||||
<rect x="36" y="5.09729" width="3.14815" height="34.8053" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="151" y="15.4779" width="3.14815" height="14.0442" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="156" y="5.70801" width="3.14815" height="33.5841" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="161" y="7.53979" width="3.14815" height="29.9204" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="166" y="15.4779" width="3.14815" height="14.0442" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="171" y="10.8982" width="3.14815" height="23.2035" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect width="3.14815" height="29.9204" rx="1.57407" transform="matrix(-1 0 0 1 149 7.5)" fill="#F3F3F3"/>
|
||||
<rect width="3.14815" height="14.0442" rx="1.57407" transform="matrix(-1 0 0 1 144 15.4381)" fill="#F3F3F3"/>
|
||||
<rect width="3.14815" height="23.2035" rx="1.57407" transform="matrix(-1 0 0 1 139 10.8584)" fill="#F3F3F3"/>
|
||||
<rect x="176" y="18.2257" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="181" y="18.2257" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="186" y="18.2257" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
<rect x="191" y="18.2257" width="3.14815" height="8.54867" rx="1.57407" fill="#F3F3F3"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 3.6 KiB |
@@ -0,0 +1,5 @@
|
||||
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<g id="Icon/Utility Icon/line/warning">
|
||||
<path id="Union" fill-rule="evenodd" clip-rule="evenodd" d="M11.8265 2.57765L19.6724 15.1369C20.0876 15.8014 20.1095 16.6395 19.7298 17.3248C19.3513 18.0101 18.6285 18.4352 17.8459 18.4352H2.15406C1.37142 18.4352 0.648615 18.0101 0.270115 17.3248C-0.109605 16.6395 -0.0876187 15.8014 0.327499 15.1369L8.17341 2.57765C8.569 1.94364 9.25275 1.56494 9.99998 1.56494C10.7472 1.56494 11.431 1.94364 11.8265 2.57765ZM17.8459 16.5589C17.9887 16.5589 18.0608 16.4685 18.0901 16.4148C18.1194 16.361 18.1585 16.2535 18.0828 16.1314L10.2369 3.57211C10.1661 3.4585 10.0574 3.44141 10 3.44141C9.94262 3.44141 9.83395 3.4585 9.76313 3.57211L1.91722 16.1314C1.84151 16.2535 1.88058 16.361 1.90988 16.4148C1.93918 16.4685 2.01122 16.5589 2.15407 16.5589H17.8459ZM9.99995 12.1893C10.5176 12.1893 10.9377 11.769 10.9377 11.2511V7.69991C10.9377 7.18195 10.5176 6.76172 9.99995 6.76172C9.48226 6.76172 9.06225 7.18195 9.06225 7.69991V11.2511C9.06225 11.7691 9.48226 12.1893 9.99995 12.1893ZM9.99996 15.6293C9.30946 15.6293 8.7497 15.0692 8.7497 14.3784C8.7497 13.6875 9.30946 13.1274 9.99996 13.1274C10.6905 13.1274 11.2502 13.6875 11.2502 14.3784C11.2502 15.0692 10.6905 15.6293 9.99996 15.6293Z" fill="#F9AC2A"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.3 KiB |
@@ -0,0 +1,3 @@
|
||||
<template>
|
||||
<div class="call-header"></div>
|
||||
</template>
|
||||
@@ -0,0 +1,82 @@
|
||||
<template>
|
||||
<div class="time">
|
||||
<div class="time-minute">{{ minute || '00' }}</div>
|
||||
<div class="time-colon">:</div>
|
||||
<div class="time-second">{{ second || '00' }}</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { limitTime, tipsRemainingTime } from '@/enums';
|
||||
|
||||
const start = defineModel();
|
||||
|
||||
const emits = defineEmits(['timeUp']);
|
||||
|
||||
const remainingTime = ref();
|
||||
const minute = ref();
|
||||
const second = ref();
|
||||
const timeInterval = ref(null);
|
||||
|
||||
const startCount = () => {
|
||||
remainingTime.value = limitTime;
|
||||
updateCountDown();
|
||||
timeInterval.value = setInterval(() => {
|
||||
updateCountDown();
|
||||
}, 1000);
|
||||
};
|
||||
const updateCountDown = () => {
|
||||
let minutes = Math.floor(remainingTime.value / 60);
|
||||
let seconds = remainingTime.value % 60;
|
||||
|
||||
// 格式化分钟和秒，确保它们是两位数
|
||||
minute.value = minutes < 10 ? '0' + minutes : minutes;
|
||||
second.value = seconds < 10 ? '0' + seconds : seconds;
|
||||
|
||||
// 剩余1分钟提示用户
|
||||
if (remainingTime.value === tipsRemainingTime) {
|
||||
ElMessage({
|
||||
type: 'warning',
|
||||
message: `This call will disconnect in ${tipsRemainingTime} seconds.`,
|
||||
duration: 3000,
|
||||
customClass: 'time-warning'
|
||||
});
|
||||
}
|
||||
// 防止倒计时变成负数
|
||||
if (remainingTime.value > 0) {
|
||||
remainingTime.value--;
|
||||
} else {
|
||||
clearInterval(timeInterval);
|
||||
emits('timeUp');
|
||||
}
|
||||
};
|
||||
watch(
|
||||
() => start.value,
|
||||
newVal => {
|
||||
timeInterval.value && clearInterval(timeInterval.value);
|
||||
if (newVal) {
|
||||
startCount();
|
||||
}
|
||||
},
|
||||
{ immediate: true }
|
||||
);
|
||||
</script>
|
||||
<style lang="less" scoped>
|
||||
.time {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
.time-minute,
|
||||
.time-second {
|
||||
width: 26px;
|
||||
height: 26px;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
border-radius: 3.848px;
|
||||
background: rgba(47, 47, 47, 0.5);
|
||||
}
|
||||
.time-colon {
|
||||
margin: 0 3px;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,23 @@
|
||||
<template>
|
||||
<div class="delay-tips">
|
||||
<span>当前发生延迟，目前延迟{{ delayTimestamp }}ms，积压{{ delayCount * 200 }}ms未发</span>
|
||||
</div>
|
||||
</template>
|
||||
<script setup>
|
||||
defineProps({
|
||||
delayTimestamp: {
|
||||
type: Number,
|
||||
defalult: 0
|
||||
},
|
||||
delayCount: {
|
||||
type: Number,
|
||||
defalult: 0
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<style lang="less" scoped>
|
||||
.delay-tips {
|
||||
font-size: 12px;
|
||||
color: #dc3545;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,36 @@
|
||||
<template>
|
||||
<div class="extra-info">
|
||||
<div class="model-version" v-if="modelVersion">模型版本: {{ modelVersion }}</div>
|
||||
<div class="web-version">前端版本: {{ webVersion }}</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
defineProps({
|
||||
modelVersion: {
|
||||
type: String,
|
||||
default: ''
|
||||
},
|
||||
webVersion: {
|
||||
type: String,
|
||||
default: ''
|
||||
}
|
||||
});
|
||||
</script>
|
||||
|
||||
<style lang="less" scoped>
|
||||
.extra-info {
|
||||
position: fixed;
|
||||
top: 62px;
|
||||
left: 4vw;
|
||||
display: flex;
|
||||
.model-version,
|
||||
.web-version {
|
||||
font-size: 12px;
|
||||
color: red;
|
||||
}
|
||||
.model-version {
|
||||
margin-right: 16px;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,67 @@
|
||||
<template>
|
||||
<div class="ideas">
|
||||
<div class="ideas-title">
|
||||
<img src="@/assets/images/ideas-icon.png " />
|
||||
<span>Convsersation ideas</span>
|
||||
</div>
|
||||
<div class="ideas-content">
|
||||
<div class="ideas-content-item" v-for="(item, index) in ideasList" :key="index">{{ item }}</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
defineProps({
|
||||
ideasList: {
|
||||
type: Array,
|
||||
default: () => []
|
||||
}
|
||||
});
|
||||
</script>
|
||||
|
||||
<style lang="less" scoped>
|
||||
.ideas {
|
||||
margin-top: 16px;
|
||||
box-shadow: 0 0 0 0.5px #e0e0e0;
|
||||
border-radius: 12px;
|
||||
padding: 18px 28px;
|
||||
&-title {
|
||||
font-size: 20px;
|
||||
font-weight: 500;
|
||||
margin-bottom: 20px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
img {
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
margin-right: 10px;
|
||||
}
|
||||
span {
|
||||
color: #171717;
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 500;
|
||||
line-height: normal;
|
||||
}
|
||||
}
|
||||
&-content {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, 1fr);
|
||||
gap: 8px;
|
||||
&-item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
border-radius: 10px;
|
||||
background: #eaefff;
|
||||
padding: 10px 24px;
|
||||
color: #7579eb;
|
||||
font-family: PingFang SC;
|
||||
font-size: 14px;
|
||||
font-style: normal;
|
||||
font-weight: 400;
|
||||
line-height: normal;
|
||||
}
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,110 @@
|
||||
<template>
|
||||
<div class="like-box">
|
||||
<div class="like-btn" @click="selectFeedbackStatus('like')">
|
||||
<img v-if="feedbackStatus === '' || feedbackStatus === 'dislike'" src="@/assets/images/zan.png" />
|
||||
<img v-else src="@/assets/images/zan-active.png" />
|
||||
</div>
|
||||
<div class="dislike-btn" @click="selectFeedbackStatus('dislike')">
|
||||
<img v-if="feedbackStatus === '' || feedbackStatus === 'like'" src="@/assets/images/cai.png" />
|
||||
<img v-else src="@/assets/images/cai-active.png" />
|
||||
</div>
|
||||
</div>
|
||||
<el-dialog
|
||||
v-model="dialogVisible"
|
||||
:title="t('feedbackDialogTitle')"
|
||||
width="400"
|
||||
:align-center="true"
|
||||
@close="cancelFeedback"
|
||||
>
|
||||
<el-input type="textarea" :rows="4" v-model="comment" />
|
||||
<div class="operate-btn">
|
||||
<el-button type="primary" :loading="submitLoading" @click="submitFeedback">确定</el-button>
|
||||
<el-button @click="cancelFeedback">取消</el-button>
|
||||
</div>
|
||||
</el-dialog>
|
||||
</template>
|
||||
<script setup>
|
||||
import { feedback } from '@/apis';
|
||||
import { useI18n } from 'vue-i18n';
|
||||
|
||||
const { t } = useI18n();
|
||||
const feedbackStatus = defineModel('feedbackStatus');
|
||||
const curResponseId = defineModel('curResponseId');
|
||||
const dialogVisible = ref(false);
|
||||
const comment = ref('');
|
||||
const submitLoading = ref(false);
|
||||
const selectFeedbackStatus = val => {
|
||||
if (!curResponseId.value) {
|
||||
return;
|
||||
}
|
||||
feedbackStatus.value = val;
|
||||
dialogVisible.value = true;
|
||||
};
|
||||
// 提交反馈
|
||||
const submitFeedback = async () => {
|
||||
submitLoading.value = true;
|
||||
const { code, message } = await feedback({
|
||||
response_id: curResponseId.value,
|
||||
rating: feedbackStatus.value,
|
||||
comment: comment.value
|
||||
});
|
||||
submitLoading.value = false;
|
||||
if (code !== 0) {
|
||||
ElMessage({
|
||||
type: 'error',
|
||||
message: message,
|
||||
duration: 3000,
|
||||
customClass: 'system-error'
|
||||
});
|
||||
return;
|
||||
}
|
||||
ElMessage.success('反馈成功');
|
||||
dialogVisible.value = false;
|
||||
setTimeout(() => {
|
||||
feedbackStatus.value = '';
|
||||
}, 2000);
|
||||
};
|
||||
const cancelFeedback = () => {
|
||||
dialogVisible.value = false;
|
||||
feedbackStatus.value = '';
|
||||
};
|
||||
</script>
|
||||
<style lang="less" scoped>
|
||||
.like-box {
|
||||
display: flex;
|
||||
margin: 0 16px;
|
||||
.like-btn,
|
||||
.dislike-btn {
|
||||
width: 26px;
|
||||
height: 26px;
|
||||
background: #f3f3f3;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
border-radius: 8px;
|
||||
cursor: pointer;
|
||||
&:hover {
|
||||
background: #d1d1d1;
|
||||
}
|
||||
img {
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
}
|
||||
}
|
||||
.dislike-btn {
|
||||
margin-left: 16px;
|
||||
}
|
||||
}
|
||||
.operate-btn {
|
||||
margin-top: 20px;
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
.el-button--primary {
|
||||
background: #647fff;
|
||||
border-color: #647fff;
|
||||
&:hover {
|
||||
border-color: #647fff;
|
||||
}
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,404 @@
|
||||
<template>
|
||||
<div class="user-config">
|
||||
<div class="user-config-title">模型配置</div>
|
||||
<div class="config-item">
|
||||
<div class="config-item-label">语音打断：</div>
|
||||
<div class="config-item-content">
|
||||
<el-switch
|
||||
v-model="configData.canStopByVoice"
|
||||
inline-prompt
|
||||
active-text="是"
|
||||
inactive-text="否"
|
||||
size="small"
|
||||
:disabled="isCalling"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<div class="config-item-label">视频画质：</div>
|
||||
<div class="config-item-content">
|
||||
<el-radio-group v-model="configData.videoQuality" :disabled="isCalling">
|
||||
<el-radio :value="true">高清</el-radio>
|
||||
<el-radio :value="false">低清</el-radio>
|
||||
</el-radio-group>
|
||||
</div>
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<div class="config-item-label">VAD阈值：</div>
|
||||
<div class="config-item-content vad-slider">
|
||||
<el-slider
|
||||
v-model="configData.vadThreshold"
|
||||
:min="0.5"
|
||||
:max="1"
|
||||
:step="0.1"
|
||||
size="small"
|
||||
:disabled="isCalling"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<!-- <div class="timbre-model">
|
||||
<div class="timbre-model-label">音色人物：</div>
|
||||
<div class="timbre-model-content">
|
||||
<el-select
|
||||
v-model="configData.timbreId"
|
||||
style="width: 100%"
|
||||
@change="handleChangePeople"
|
||||
clearable
|
||||
placeholder="请选择"
|
||||
>
|
||||
<el-option v-for="item in peopleList" :key="item.id" :value="item.id" :label="item.name">
|
||||
{{ item.name }}
|
||||
</el-option>
|
||||
</el-select>
|
||||
</div>
|
||||
</div> -->
|
||||
<div class="prompt-item">
|
||||
<div class="prompt-item-label">Assistant_prompt：</div>
|
||||
<div class="prompt-item-content">
|
||||
<el-input
|
||||
type="textarea"
|
||||
:rows="3"
|
||||
v-model="configData.assistantPrompt"
|
||||
resize="none"
|
||||
:disabled="isCalling"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<div class="config-item-label">使用语音prompt：</div>
|
||||
<div class="config-item-content">
|
||||
<el-switch
|
||||
v-model="configData.useAudioPrompt"
|
||||
inline-prompt
|
||||
active-text="是"
|
||||
inactive-text="否"
|
||||
size="small"
|
||||
:disabled="isCalling"
|
||||
@change="handleSelectUseAudioPrompt"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="voice-prompt-box">
|
||||
<div class="prompt-item" v-if="configData.useAudioPrompt">
|
||||
<div class="prompt-item-label">Voice_clone_prompt：</div>
|
||||
<div class="prompt-item-content">
|
||||
<el-input
|
||||
type="textarea"
|
||||
:rows="8"
|
||||
v-model="configData.voiceClonePrompt"
|
||||
resize="none"
|
||||
:disabled="isCalling"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="timbre-config" v-if="configData.useAudioPrompt">
|
||||
<div class="timbre-config-label">音色选择：</div>
|
||||
<div class="timbre-config-content">
|
||||
<el-checkbox-group v-model="configData.timbre" @change="handleSelectTimbre" :disabled="isCalling">
|
||||
<el-checkbox :value="1" label="Default Audio"></el-checkbox>
|
||||
<el-upload
|
||||
v-model:file-list="fileList"
|
||||
action=""
|
||||
:multiple="false"
|
||||
:on-change="handleChangeFile"
|
||||
:auto-upload="false"
|
||||
:show-file-list="false"
|
||||
:disabled="isCalling"
|
||||
accept="audio/*"
|
||||
>
|
||||
<el-checkbox :value="2">
|
||||
<!-- <span>Customization: Upload Audio</span> -->
|
||||
<span>Customization</span>
|
||||
<SvgIcon name="upload" className="checkbox-icon" />
|
||||
</el-checkbox>
|
||||
</el-upload>
|
||||
</el-checkbox-group>
|
||||
</div>
|
||||
</div>
|
||||
<div class="file-content" v-if="fileName">
|
||||
<SvgIcon name="document" class="document-icon" />
|
||||
<span class="file-name">{{ fileName }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
const isCalling = defineModel('isCalling');
|
||||
const type = defineModel('type');
|
||||
|
||||
let defaultVoiceClonePrompt =
|
||||
'你是一个AI助手。你能接受视频，音频和文本输入并输出语音和文本。模仿输入音频中的声音特征。';
|
||||
let defaultAssistantPrompt = '作为助手，你将使用这种声音风格说话。';
|
||||
|
||||
const fileList = ref([]);
|
||||
const fileName = ref('');
|
||||
|
||||
const configData = ref({
|
||||
canStopByVoice: false,
|
||||
videoQuality: false,
|
||||
useAudioPrompt: true,
|
||||
vadThreshold: 0.8,
|
||||
voiceClonePrompt: defaultVoiceClonePrompt,
|
||||
assistantPrompt: defaultAssistantPrompt,
|
||||
timbre: [1],
|
||||
audioFormat: 'mp3',
|
||||
base64Str: '',
|
||||
timbreId: ''
|
||||
});
|
||||
|
||||
const peopleList = [
|
||||
{
|
||||
id: 1,
|
||||
name: 'Trump',
|
||||
voiceClonePrompt: '',
|
||||
assistantPrompt: ''
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
name: '说相声',
|
||||
voiceClonePrompt: '克隆音频提示中的音色以生成语音',
|
||||
assistantPrompt: '请角色扮演这段音频，请以相声演员的口吻说话'
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
name: '默认',
|
||||
voiceClonePrompt: defaultVoiceClonePrompt,
|
||||
assistantPrompt: defaultAssistantPrompt
|
||||
}
|
||||
];
|
||||
watch(
|
||||
() => type.value,
|
||||
val => {
|
||||
if (val === 'video') {
|
||||
console.log('val: ', val);
|
||||
defaultVoiceClonePrompt =
|
||||
'你是一个AI助手。你能接受视频，音频和文本输入并输出语音和文本。模仿输入音频中的声音特征。';
|
||||
defaultAssistantPrompt = '作为助手，你将使用这种声音风格说话。';
|
||||
} else {
|
||||
defaultVoiceClonePrompt = '克隆音频提示中的音色以生成语音。';
|
||||
defaultAssistantPrompt = 'Your task is to be a helpful assistant using this voice pattern.';
|
||||
}
|
||||
configData.value.voiceClonePrompt = defaultVoiceClonePrompt;
|
||||
configData.value.assistantPrompt = defaultAssistantPrompt;
|
||||
},
|
||||
{ immediate: true }
|
||||
);
|
||||
onMounted(() => {
|
||||
handleSetStorage();
|
||||
});
|
||||
const handleSelectTimbre = e => {
|
||||
if (e.length > 1) {
|
||||
const val = e[e.length - 1];
|
||||
configData.value.timbre = [val];
|
||||
// 默认音色
|
||||
if (val === 1) {
|
||||
configData.value.audioFormat = 'mp3';
|
||||
configData.value.base64Str = '';
|
||||
fileList.value = [];
|
||||
fileName.value = '';
|
||||
}
|
||||
}
|
||||
};
|
||||
const handleChangeFile = file => {
|
||||
if (isAudio(file) && sizeNotExceed(file)) {
|
||||
fileList.value = [file];
|
||||
fileName.value = file.name;
|
||||
configData.value.timbre = [2];
|
||||
handleUpload();
|
||||
} else {
|
||||
ElMessage.error('Please upload audio file and size not exceed 10MB');
|
||||
}
|
||||
};
|
||||
const isAudio = file => {
|
||||
return file.raw.type.includes('audio');
|
||||
};
|
||||
const sizeNotExceed = file => {
|
||||
return file.size / 1024 / 1024 <= 10;
|
||||
};
|
||||
const handleUpload = async () => {
|
||||
const file = fileList.value[0].raw;
|
||||
if (file) {
|
||||
const reader = new FileReader();
|
||||
reader.onload = e => {
|
||||
const base64String = e.target.result.split(',')[1];
|
||||
configData.value.audioFormat = file.name.split('.')[1];
|
||||
configData.value.base64Str = base64String;
|
||||
};
|
||||
reader.readAsDataURL(file);
|
||||
}
|
||||
};
|
||||
const handleSelectUseAudioPrompt = val => {
|
||||
if (val) {
|
||||
configData.value.voiceClonePrompt = defaultVoiceClonePrompt;
|
||||
configData.value.assistantPrompt = defaultAssistantPrompt;
|
||||
}
|
||||
};
|
||||
// 配置发生变化，更新到localstorage中
|
||||
watch(configData.value, () => {
|
||||
handleSetStorage();
|
||||
});
|
||||
const handleSetStorage = () => {
|
||||
const { timbre, canStopByVoice, ...others } = configData.value;
|
||||
const defaultConfigData = {
|
||||
canStopByVoice,
|
||||
...others
|
||||
};
|
||||
localStorage.setItem('configData', JSON.stringify(defaultConfigData));
|
||||
localStorage.setItem('canStopByVoice', canStopByVoice);
|
||||
};
|
||||
const handleChangePeople = val => {
|
||||
console.log('val: ', val);
|
||||
if (!val) {
|
||||
return;
|
||||
}
|
||||
const index = peopleList.findIndex(item => item.id === val);
|
||||
configData.value.voiceClonePrompt = peopleList[index].voiceClonePrompt;
|
||||
configData.value.assistantPrompt = peopleList[index].assistantPrompt;
|
||||
configData.value.timbre = [1];
|
||||
};
|
||||
</script>
|
||||
<style lang="less">
|
||||
.user-config {
|
||||
&-title {
|
||||
height: 61px;
|
||||
padding: 18px 18px 0;
|
||||
color: rgba(23, 23, 23, 0.9);
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 500;
|
||||
line-height: normal;
|
||||
}
|
||||
.config-item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
width: 100%;
|
||||
padding: 0 0 0 18px;
|
||||
margin-bottom: 12px;
|
||||
&-label {
|
||||
width: 120px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
&-content {
|
||||
flex: 1;
|
||||
margin-left: 16px;
|
||||
.el-radio-group {
|
||||
.el-radio {
|
||||
width: 50px;
|
||||
}
|
||||
}
|
||||
}
|
||||
&-content.vad-slider {
|
||||
width: 80%;
|
||||
padding-left: 7px;
|
||||
margin-right: 20px;
|
||||
.el-slider__button {
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
}
|
||||
}
|
||||
}
|
||||
.timbre-config {
|
||||
padding: 0 0 0 18px;
|
||||
&-label {
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
&-content {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
.el-checkbox-group {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
flex: 1;
|
||||
> .el-checkbox {
|
||||
margin-right: 12px;
|
||||
}
|
||||
}
|
||||
.el-checkbox {
|
||||
padding: 8px 16px;
|
||||
border-radius: 10px;
|
||||
background: #eaefff;
|
||||
margin-bottom: 12px;
|
||||
height: 40px;
|
||||
.el-checkbox__input {
|
||||
.el-checkbox__inner {
|
||||
border: 1px solid #4dc100;
|
||||
}
|
||||
}
|
||||
.el-checkbox__input.is-checked {
|
||||
.el-checkbox__inner {
|
||||
background: #4dc100;
|
||||
}
|
||||
}
|
||||
.el-checkbox__input.is-checked.is-disabled {
|
||||
.el-checkbox__inner::after {
|
||||
border-color: #ffffff;
|
||||
}
|
||||
}
|
||||
}
|
||||
.el-checkbox__label {
|
||||
color: #7579eb !important;
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 400;
|
||||
line-height: normal;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
.checkbox-icon {
|
||||
margin-left: 4px;
|
||||
}
|
||||
}
|
||||
.el-checkbox + .el-checkbox {
|
||||
margin-left: 12px;
|
||||
}
|
||||
}
|
||||
}
|
||||
.prompt-item {
|
||||
// padding: 0 0 0 18px;
|
||||
margin-bottom: 12px;
|
||||
&-label {
|
||||
// margin-bottom: 16px;
|
||||
}
|
||||
}
|
||||
.file-content {
|
||||
padding: 0 0 0 18px;
|
||||
font-size: 14px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
.document-icon {
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
margin-right: 4px;
|
||||
}
|
||||
.file-name {
|
||||
flex: 1;
|
||||
overflow: hidden;
|
||||
white-space: nowrap;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
}
|
||||
.timbre-model {
|
||||
padding: 0 0 0 18px;
|
||||
margin-bottom: 12px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
&-label {
|
||||
width: 120px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
&-content {
|
||||
flex: 1;
|
||||
margin-left: 16px;
|
||||
}
|
||||
}
|
||||
.voice-prompt-box {
|
||||
border: 1px solid #eaefff;
|
||||
margin-left: 18px;
|
||||
padding: 12px;
|
||||
width: 50%;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,456 @@
|
||||
<template>
|
||||
<div :class="`user-config ${t('modelConfigTitle') === '模型配置' ? '' : 'en-user-config'}`">
|
||||
<div class="user-config-title">{{ t('modelConfigTitle') }}</div>
|
||||
<div class="config-item">
|
||||
<div class="config-item-label">
|
||||
<span>{{ t('audioInterruptionBtn') }}</span>
|
||||
<el-tooltip class="box-item" effect="dark" :content="t('audioInterruptionTips')" placement="top">
|
||||
<SvgIcon name="question" class="question-icon" /> </el-tooltip
|
||||
>:
|
||||
</div>
|
||||
<div class="config-item-content">
|
||||
<el-switch
|
||||
v-model="configData.canStopByVoice"
|
||||
inline-prompt
|
||||
:active-text="t('yes')"
|
||||
:inactive-text="t('no')"
|
||||
size="small"
|
||||
:disabled="isCalling"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="config-item" v-if="type === 'video'">
|
||||
<div class="config-item-label">
|
||||
<span>{{ t('videoQualityBtn') }}</span>
|
||||
<el-tooltip class="box-item" effect="dark" :content="t('videoQualityTips')" placement="top">
|
||||
<SvgIcon name="question" class="question-icon" /> </el-tooltip
|
||||
>:
|
||||
</div>
|
||||
<div class="config-item-content">
|
||||
<el-switch
|
||||
v-model="configData.videoQuality"
|
||||
inline-prompt
|
||||
:active-text="t('yes')"
|
||||
:inactive-text="t('no')"
|
||||
size="small"
|
||||
:disabled="isCalling"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<div class="config-item-label">
|
||||
<span>{{ t('vadThresholdBtn') }}</span>
|
||||
<el-tooltip class="box-item" effect="dark" :content="t('vadThresholdTips')" placement="top">
|
||||
<SvgIcon name="question" class="question-icon" /> </el-tooltip
|
||||
>:
|
||||
</div>
|
||||
<div class="config-item-content vad-slider">
|
||||
<el-slider
|
||||
v-model="configData.vadThreshold"
|
||||
:min="0.5"
|
||||
:max="1"
|
||||
:step="0.1"
|
||||
size="small"
|
||||
:disabled="isCalling"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="prompt-item" v-if="type === 'voice'">
|
||||
<div class="prompt-item-label">
|
||||
<span>{{ t('assistantPromptBtn') }}</span>
|
||||
<el-tooltip class="box-item" effect="dark" :content="t('assistantPromptTips')" placement="top">
|
||||
<SvgIcon name="question" class="question-icon" /> </el-tooltip
|
||||
>:
|
||||
</div>
|
||||
<div class="prompt-item-content">
|
||||
<el-input
|
||||
type="textarea"
|
||||
:rows="3"
|
||||
v-model="configData.assistantPrompt"
|
||||
resize="none"
|
||||
:disabled="isCalling"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<!-- <div class="config-item">
|
||||
<div class="config-item-label">{{ t('useVoicePromptBtn') }}:</div>
|
||||
<div class="config-item-content">
|
||||
<el-switch
|
||||
v-model="configData.useAudioPrompt"
|
||||
inline-prompt
|
||||
:active-text="t('yes')"
|
||||
:inactive-text="t('no')"
|
||||
size="small"
|
||||
:disabled="isCalling"
|
||||
@change="handleSelectUseAudioPrompt"
|
||||
/>
|
||||
</div>
|
||||
</div> -->
|
||||
<div class="timbre-model">
|
||||
<div class="timbre-model-label">
|
||||
<span>{{ t('toneColorOptions') }}</span>
|
||||
<el-tooltip class="box-item" effect="dark" :content="t('toneColorOptionsTips')" placement="top">
|
||||
<SvgIcon name="question" class="question-icon" /> </el-tooltip
|
||||
>:
|
||||
</div>
|
||||
<div class="timbre-model-content">
|
||||
<el-select
|
||||
v-model="configData.useAudioPrompt"
|
||||
style="width: 100%"
|
||||
@change="handleChangePeople"
|
||||
placeholder="请选择"
|
||||
:disabled="isCalling"
|
||||
>
|
||||
<el-option :value="0" :label="t('nullOption')">{{ t('nullOption') }}</el-option>
|
||||
<el-option :value="1" :label="t('defaultOption')">{{ t('defaultOption') }}</el-option>
|
||||
<el-option :value="2" :label="t('femaleOption')">{{ t('femaleOption') }}</el-option>
|
||||
<el-option :value="3" :label="t('maleOption')">{{ t('maleOption') }}</el-option>
|
||||
</el-select>
|
||||
</div>
|
||||
</div>
|
||||
<!-- <div class="prompt-item">
|
||||
<div class="prompt-item-label">
|
||||
<span>{{ t('voiceClonePromptInput') }}</span>
|
||||
<el-tooltip class="box-item" effect="dark" :content="t('voiceClonePromptTips')" placement="top">
|
||||
<SvgIcon name="question" class="question-icon" /> </el-tooltip
|
||||
>:
|
||||
</div>
|
||||
<div class="prompt-item-content">
|
||||
<el-input
|
||||
type="textarea"
|
||||
:rows="3"
|
||||
v-model="configData.voiceClonePrompt"
|
||||
resize="none"
|
||||
:disabled="true"
|
||||
/>
|
||||
</div>
|
||||
</div> -->
|
||||
<!-- <div class="timbre-config" v-if="configData.useAudioPrompt">
|
||||
<div class="timbre-config-label">{{ t('audioChoiceBtn') }}:</div>
|
||||
<div class="timbre-config-content">
|
||||
<el-checkbox-group v-model="configData.timbre" @change="handleSelectTimbre" :disabled="isCalling">
|
||||
<el-checkbox :value="1" :label="t('defaultAudioBtn')"></el-checkbox>
|
||||
<el-upload
|
||||
v-model:file-list="fileList"
|
||||
action=""
|
||||
:multiple="false"
|
||||
:on-change="handleChangeFile"
|
||||
:auto-upload="false"
|
||||
:show-file-list="false"
|
||||
:disabled="isCalling"
|
||||
accept="audio/*"
|
||||
>
|
||||
<el-checkbox :value="2">
|
||||
<span>{{ t('customizationBtn') }}</span>
|
||||
<SvgIcon name="upload" className="checkbox-icon" />
|
||||
</el-checkbox>
|
||||
</el-upload>
|
||||
</el-checkbox-group>
|
||||
</div>
|
||||
</div>
|
||||
<div class="file-content" v-if="fileName">
|
||||
<SvgIcon name="document" class="document-icon" />
|
||||
<span class="file-name">{{ fileName }}</span>
|
||||
</div> -->
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
const isCalling = defineModel('isCalling');
|
||||
const type = defineModel('type');
|
||||
import { useI18n } from 'vue-i18n';
|
||||
|
||||
const { t, locale } = useI18n();
|
||||
|
||||
let defaultVoiceClonePrompt =
|
||||
'你是一个AI助手。你能接受视频，音频和文本输入并输出语音和文本。模仿输入音频中的声音特征。';
|
||||
let defaultAssistantPrompt = '';
|
||||
|
||||
const fileList = ref([]);
|
||||
const fileName = ref('');
|
||||
|
||||
const configData = ref({
|
||||
canStopByVoice: false,
|
||||
videoQuality: false,
|
||||
useAudioPrompt: 1,
|
||||
vadThreshold: 0.8,
|
||||
voiceClonePrompt: defaultVoiceClonePrompt,
|
||||
assistantPrompt: defaultAssistantPrompt,
|
||||
timbre: [1],
|
||||
audioFormat: 'mp3',
|
||||
base64Str: ''
|
||||
});
|
||||
|
||||
// let peopleList = [];
|
||||
// watch(
|
||||
// () => type.value,
|
||||
// val => {
|
||||
// console.log('val: ', val);
|
||||
// if (val === 'video') {
|
||||
// defaultVoiceClonePrompt =
|
||||
// '你是一个AI助手。你能接受视频，音频和文本输入并输出语音和文本。模仿输入音频中的声音特征。';
|
||||
// defaultAssistantPrompt = '作为助手，你将使用这种声音风格说话。';
|
||||
// } else {
|
||||
// defaultVoiceClonePrompt = '克隆音频提示中的音色以生成语音。';
|
||||
// defaultAssistantPrompt = 'Your task is to be a helpful assistant using this voice pattern.';
|
||||
// }
|
||||
// configData.value.voiceClonePrompt = defaultVoiceClonePrompt;
|
||||
// configData.value.assistantPrompt = defaultAssistantPrompt;
|
||||
// },
|
||||
// { immediate: true }
|
||||
// );
|
||||
watch(
|
||||
locale,
|
||||
(newLocale, oldLocale) => {
|
||||
console.log(`Language switched from ${oldLocale} to ${newLocale}`);
|
||||
if (newLocale === 'zh' && type.value === 'video') {
|
||||
defaultAssistantPrompt = '作为助手，你将使用这种声音风格说话。';
|
||||
} else if (newLocale === 'zh' && type.value === 'voice') {
|
||||
defaultAssistantPrompt = '作为助手，你将使用这种声音风格说话。';
|
||||
} else if (newLocale === 'en' && type.value === 'video') {
|
||||
defaultAssistantPrompt = 'As an assistant, you will speak using this voice style.';
|
||||
} else {
|
||||
defaultAssistantPrompt = 'As an assistant, you will speak using this voice style.';
|
||||
}
|
||||
configData.value.assistantPrompt = defaultAssistantPrompt;
|
||||
},
|
||||
{ immediate: true }
|
||||
);
|
||||
onMounted(() => {
|
||||
handleSetStorage();
|
||||
});
|
||||
const handleSelectTimbre = e => {
|
||||
if (e.length > 1) {
|
||||
const val = e[e.length - 1];
|
||||
configData.value.timbre = [val];
|
||||
// 默认音色
|
||||
if (val === 1) {
|
||||
configData.value.audioFormat = 'mp3';
|
||||
configData.value.base64Str = '';
|
||||
fileList.value = [];
|
||||
fileName.value = '';
|
||||
}
|
||||
}
|
||||
};
|
||||
const handleChangeFile = file => {
|
||||
if (isAudio(file) && sizeNotExceed(file)) {
|
||||
fileList.value = [file];
|
||||
fileName.value = file.name;
|
||||
configData.value.timbre = [2];
|
||||
handleUpload();
|
||||
} else {
|
||||
ElMessage.error('Please upload audio file and size not exceed 10MB');
|
||||
}
|
||||
};
|
||||
const isAudio = file => {
|
||||
return file.raw.type.includes('audio');
|
||||
};
|
||||
const sizeNotExceed = file => {
|
||||
return file.size / 1024 / 1024 <= 10;
|
||||
};
|
||||
const handleUpload = async () => {
|
||||
const file = fileList.value[0].raw;
|
||||
if (file) {
|
||||
const reader = new FileReader();
|
||||
reader.onload = e => {
|
||||
const base64String = e.target.result.split(',')[1];
|
||||
configData.value.audioFormat = file.name.split('.')[1];
|
||||
configData.value.base64Str = base64String;
|
||||
};
|
||||
reader.readAsDataURL(file);
|
||||
}
|
||||
};
|
||||
const handleSelectUseAudioPrompt = val => {
|
||||
if (val) {
|
||||
configData.value.voiceClonePrompt = defaultVoiceClonePrompt;
|
||||
configData.value.assistantPrompt = defaultAssistantPrompt;
|
||||
}
|
||||
};
|
||||
// 配置发生变化，更新到localstorage中
|
||||
watch(configData.value, () => {
|
||||
handleSetStorage();
|
||||
});
|
||||
const handleSetStorage = () => {
|
||||
const { timbre, canStopByVoice, ...others } = configData.value;
|
||||
const defaultConfigData = {
|
||||
canStopByVoice,
|
||||
...others
|
||||
};
|
||||
localStorage.setItem('configData', JSON.stringify(defaultConfigData));
|
||||
localStorage.setItem('canStopByVoice', canStopByVoice);
|
||||
};
|
||||
const handleChangePeople = val => {
|
||||
console.log('val: ', val);
|
||||
// const index = peopleList.findIndex(item => item.id === val);
|
||||
configData.value.voiceClonePrompt = defaultVoiceClonePrompt;
|
||||
configData.value.assistantPrompt = defaultAssistantPrompt;
|
||||
configData.value.timbre = [1];
|
||||
};
|
||||
</script>
|
||||
<style lang="less" scoped>
|
||||
.user-config {
|
||||
&-title {
|
||||
height: 61px;
|
||||
padding: 18px 18px 0;
|
||||
color: rgba(23, 23, 23, 0.9);
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 500;
|
||||
line-height: normal;
|
||||
}
|
||||
.config-item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
width: 100%;
|
||||
padding: 0 0 0 18px;
|
||||
margin-bottom: 20px;
|
||||
&-label {
|
||||
width: 120px;
|
||||
flex-shrink: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
&-content {
|
||||
flex: 1;
|
||||
margin-left: 16px;
|
||||
.el-radio-group {
|
||||
.el-radio {
|
||||
width: 50px;
|
||||
}
|
||||
}
|
||||
}
|
||||
&-content.vad-slider {
|
||||
width: 80%;
|
||||
padding-left: 7px;
|
||||
margin-right: 20px;
|
||||
.el-slider__button {
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
}
|
||||
}
|
||||
}
|
||||
.timbre-config {
|
||||
padding: 0 0 0 18px;
|
||||
&-label {
|
||||
margin-bottom: 20px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
&-content {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
.el-checkbox-group {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
flex: 1;
|
||||
> .el-checkbox {
|
||||
margin-right: 12px;
|
||||
}
|
||||
}
|
||||
.el-checkbox {
|
||||
padding: 8px 16px;
|
||||
border-radius: 10px;
|
||||
background: #eaefff;
|
||||
margin-bottom: 12px;
|
||||
height: 40px;
|
||||
.el-checkbox__input {
|
||||
.el-checkbox__inner {
|
||||
border: 1px solid #4dc100;
|
||||
}
|
||||
}
|
||||
.el-checkbox__input.is-checked {
|
||||
.el-checkbox__inner {
|
||||
background: #4dc100;
|
||||
}
|
||||
}
|
||||
.el-checkbox__input.is-checked.is-disabled {
|
||||
.el-checkbox__inner::after {
|
||||
border-color: #ffffff;
|
||||
}
|
||||
}
|
||||
}
|
||||
.el-checkbox__label {
|
||||
color: #7579eb !important;
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 400;
|
||||
line-height: normal;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
.checkbox-icon {
|
||||
margin-left: 4px;
|
||||
}
|
||||
}
|
||||
.el-checkbox + .el-checkbox {
|
||||
margin-left: 12px;
|
||||
}
|
||||
}
|
||||
}
|
||||
.prompt-item {
|
||||
padding: 0 0 0 18px;
|
||||
margin-bottom: 20px;
|
||||
&-label {
|
||||
// margin-bottom: 16px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
}
|
||||
.file-content {
|
||||
padding: 0 0 0 18px;
|
||||
font-size: 14px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
.document-icon {
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
margin-right: 4px;
|
||||
}
|
||||
.file-name {
|
||||
flex: 1;
|
||||
overflow: hidden;
|
||||
white-space: nowrap;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
}
|
||||
.timbre-model {
|
||||
padding: 0 0 0 18px;
|
||||
margin-bottom: 20px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
&-label {
|
||||
width: 120px;
|
||||
flex-shrink: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
&-content {
|
||||
flex: 1;
|
||||
margin-left: 16px;
|
||||
}
|
||||
}
|
||||
}
|
||||
.en-user-config {
|
||||
.config-item-label {
|
||||
width: 160px;
|
||||
}
|
||||
.timbre-model-label {
|
||||
width: 160px;
|
||||
}
|
||||
}
|
||||
.question-icon {
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
cursor: pointer;
|
||||
margin-left: 6px;
|
||||
}
|
||||
</style>
|
||||
<style lang="less">
|
||||
.el-switch--small .el-switch__core {
|
||||
min-width: 50px;
|
||||
}
|
||||
.el-popper.is-dark {
|
||||
max-width: 300px;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,91 @@
|
||||
<template>
|
||||
<div class="output-area">
|
||||
<div
|
||||
:class="`output-area-item ${item.type === 'USER' ? 'user-item' : 'bot-item'}`"
|
||||
:key="index"
|
||||
v-for="(item, index) in outputData"
|
||||
>
|
||||
<div v-if="item.type === 'USER'" class="user-input">
|
||||
<audio v-if="item.audio" :src="item.audio" controls></audio>
|
||||
</div>
|
||||
<div v-else class="bot-output">
|
||||
<div class="output-item">{{ item.text }}</div>
|
||||
<audio v-if="item.audio" :src="item.audio" controls></audio>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
const props = defineProps({
|
||||
outputData: {
|
||||
type: Array,
|
||||
default: () => []
|
||||
},
|
||||
containerClass: {
|
||||
type: String,
|
||||
default: ''
|
||||
}
|
||||
});
|
||||
watch(
|
||||
() => props.outputData,
|
||||
newVal => {
|
||||
nextTick(() => {
|
||||
if (newVal && props.containerClass) {
|
||||
let dom = document.querySelector(`.${props.containerClass}`);
|
||||
if (dom) {
|
||||
dom.scrollTop = dom.scrollHeight;
|
||||
}
|
||||
}
|
||||
});
|
||||
},
|
||||
{ deep: true }
|
||||
);
|
||||
</script>
|
||||
|
||||
<style lang="less" scoped>
|
||||
.output-area {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
&-item {
|
||||
width: fit-content;
|
||||
}
|
||||
&-item + &-item {
|
||||
margin-top: 16px;
|
||||
}
|
||||
&-item.user-item {
|
||||
align-self: flex-end;
|
||||
.user-input {
|
||||
}
|
||||
}
|
||||
&-item.bot-item {
|
||||
align-self: flex-start;
|
||||
width: 100%;
|
||||
.bot-output {
|
||||
width: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
.output-item {
|
||||
padding: 8px 24px;
|
||||
border-radius: 10px;
|
||||
color: #202224;
|
||||
background: #f3f3f3;
|
||||
max-width: 90%;
|
||||
width: fit-content;
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 400;
|
||||
line-height: normal;
|
||||
word-break: break-all;
|
||||
word-wrap: break-word;
|
||||
white-space: pre-wrap;
|
||||
display: inline-block;
|
||||
}
|
||||
.output-item + audio {
|
||||
margin-top: 16px;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,122 @@
|
||||
<template>
|
||||
<div class="select-timbre">
|
||||
<el-checkbox-group v-model="timbre" @change="handleSelectTimbre" :disabled="disabled">
|
||||
<el-checkbox :value="1" label="Default Audio"></el-checkbox>
|
||||
<!-- <el-upload
|
||||
v-model:file-list="fileList"
|
||||
action=""
|
||||
:multiple="false"
|
||||
:on-change="handleChangeFile"
|
||||
:auto-upload="false"
|
||||
:show-file-list="false"
|
||||
:disabled="disabled"
|
||||
accept="audio/*"
|
||||
>
|
||||
<el-checkbox :value="2">
|
||||
<span>Customization: Upload Audio</span>
|
||||
<SvgIcon name="upload" className="checkbox-icon" />
|
||||
</el-checkbox>
|
||||
</el-upload> -->
|
||||
</el-checkbox-group>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
const timbre = defineModel('timbre');
|
||||
const audioData = defineModel('audioData');
|
||||
const disabled = defineModel('disabled');
|
||||
const fileList = ref([]);
|
||||
|
||||
const handleSelectTimbre = e => {
|
||||
if (e.length > 1) {
|
||||
const val = e[e.length - 1];
|
||||
timbre.value = [val];
|
||||
// 默认音色
|
||||
if (val === 1) {
|
||||
audioData.value = {
|
||||
base64Str: '',
|
||||
type: 'mp3'
|
||||
};
|
||||
}
|
||||
}
|
||||
};
|
||||
const handleChangeFile = file => {
|
||||
if (isAudio(file) && sizeNotExceed(file)) {
|
||||
fileList.value = [file];
|
||||
timbre.value = [2];
|
||||
handleUpload();
|
||||
} else {
|
||||
ElMessage.error('Please upload audio file and size not exceed 1MB');
|
||||
}
|
||||
};
|
||||
const isAudio = file => {
|
||||
return file.name.endsWith('.mp3') || file.name.endsWith('.wav');
|
||||
};
|
||||
const sizeNotExceed = file => {
|
||||
return file.size / 1024 / 1024 <= 1;
|
||||
};
|
||||
const handleUpload = async () => {
|
||||
const file = fileList.value[0].raw;
|
||||
if (file) {
|
||||
const reader = new FileReader();
|
||||
reader.onload = e => {
|
||||
const base64String = e.target.result.split(',')[1];
|
||||
audioData.value = {
|
||||
base64Str: base64String,
|
||||
type: file.name.split('.')[1]
|
||||
};
|
||||
};
|
||||
reader.readAsDataURL(file);
|
||||
}
|
||||
};
|
||||
</script>
|
||||
<style lang="less">
|
||||
.select-timbre {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
.el-checkbox-group {
|
||||
display: flex;
|
||||
> .el-checkbox {
|
||||
margin-right: 12px;
|
||||
}
|
||||
}
|
||||
.el-checkbox {
|
||||
padding: 8px 16px;
|
||||
border-radius: 10px;
|
||||
background: #eaefff;
|
||||
margin-right: 0;
|
||||
height: 40px;
|
||||
.el-checkbox__input {
|
||||
.el-checkbox__inner {
|
||||
border: 1px solid #4dc100;
|
||||
}
|
||||
}
|
||||
.el-checkbox__input.is-checked {
|
||||
.el-checkbox__inner {
|
||||
background: #4dc100;
|
||||
}
|
||||
}
|
||||
.el-checkbox__input.is-checked.is-disabled {
|
||||
.el-checkbox__inner::after {
|
||||
border-color: #ffffff;
|
||||
}
|
||||
}
|
||||
}
|
||||
.el-checkbox__label {
|
||||
color: #7579eb !important;
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 400;
|
||||
line-height: normal;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
.checkbox-icon {
|
||||
margin-left: 4px;
|
||||
}
|
||||
}
|
||||
.el-checkbox + .el-checkbox {
|
||||
margin-left: 12px;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,67 @@
|
||||
<template>
|
||||
<div :class="`skip-btn ${disabled ? 'disabled-btn' : ''}`">
|
||||
<div class="pause-icon">
|
||||
<SvgIcon name="pause" className="pause-svg" />
|
||||
</div>
|
||||
<span class="btn-text">{{ t('skipMessageBtn') }}</span>
|
||||
</div>
|
||||
</template>
|
||||
<script setup>
|
||||
import { useI18n } from 'vue-i18n';
|
||||
|
||||
const { t } = useI18n();
|
||||
defineProps({
|
||||
disabled: {
|
||||
type: Boolean,
|
||||
default: false
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<style lang="less">
|
||||
.skip-btn {
|
||||
flex-shrink: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
padding: 8px 14px 8px 10px;
|
||||
border-radius: 90px;
|
||||
background: #5865f2;
|
||||
cursor: pointer;
|
||||
user-select: none;
|
||||
.pause-icon {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
background: #ffffff;
|
||||
border-radius: 50%;
|
||||
margin-right: 8px;
|
||||
.pause-svg {
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
color: #5865f2;
|
||||
}
|
||||
}
|
||||
.btn-text {
|
||||
color: #fff;
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 400;
|
||||
line-height: normal;
|
||||
}
|
||||
}
|
||||
.disabled-btn {
|
||||
cursor: not-allowed;
|
||||
background: #f3f3f3;
|
||||
.pause-icon {
|
||||
background: #d1d1d1;
|
||||
.pause-svg {
|
||||
color: #ffffff;
|
||||
}
|
||||
}
|
||||
.btn-text {
|
||||
color: #d1d1d1;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,39 @@
|
||||
<template>
|
||||
<svg :class="iconClass" v-html="content"></svg>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
const props = defineProps({
|
||||
name: {
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
className: {
|
||||
type: String,
|
||||
default: ''
|
||||
}
|
||||
});
|
||||
|
||||
const content = ref('');
|
||||
|
||||
const iconClass = computed(() => ['svg-icon', props.className]);
|
||||
onMounted(() => {
|
||||
import(`@/assets/svg/${props.name}.svg`)
|
||||
.then(module => {
|
||||
fetch(module.default)
|
||||
.then(response => response.text())
|
||||
.then(svg => {
|
||||
content.value = svg;
|
||||
});
|
||||
})
|
||||
.catch(error => {
|
||||
console.error(`Error loading SVG icon: ${props.name}`, error);
|
||||
});
|
||||
});
|
||||
</script>
|
||||
<style lang="less" scoped>
|
||||
.svg-icon {
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,138 @@
|
||||
<template>
|
||||
<div class="bars" id="bars" :style="boxStyle">
|
||||
<!-- 柱形条 -->
|
||||
<div class="bar" v-for="(item, index) in defaultList" :key="index" :style="itemAttr(item)"></div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
const props = defineProps({
|
||||
analyser: {
|
||||
type: Object
|
||||
},
|
||||
dataArray: {
|
||||
type: [Array, Uint8Array]
|
||||
},
|
||||
isCalling: {
|
||||
type: Boolean,
|
||||
default: false
|
||||
},
|
||||
isPlaying: {
|
||||
type: Boolean,
|
||||
default: false
|
||||
},
|
||||
// 容器高度
|
||||
boxStyle: {
|
||||
type: Object,
|
||||
default: () => {
|
||||
return {
|
||||
height: '80px'
|
||||
};
|
||||
}
|
||||
},
|
||||
// 柱形条宽度
|
||||
itemStyle: {
|
||||
type: Object,
|
||||
default: () => {
|
||||
return {
|
||||
width: '6px',
|
||||
margin: '0 2px',
|
||||
borderRadius: '5px'
|
||||
};
|
||||
}
|
||||
},
|
||||
configList: {
|
||||
type: Array,
|
||||
default: () => []
|
||||
}
|
||||
});
|
||||
const animationFrameId = ref();
|
||||
const defaultList = ref([]);
|
||||
const bgColor = ref('#4c5cf8');
|
||||
const itemAttr = computed(() => item => {
|
||||
return {
|
||||
height: item + 'px',
|
||||
...props.itemStyle
|
||||
};
|
||||
});
|
||||
watch(
|
||||
() => props.dataArray,
|
||||
newVal => {
|
||||
if (newVal && props.isCalling) {
|
||||
console.log('draw');
|
||||
drawBars();
|
||||
} else {
|
||||
console.log('stop');
|
||||
stopDraw();
|
||||
}
|
||||
}
|
||||
);
|
||||
watch(
|
||||
() => props.configList,
|
||||
newVal => {
|
||||
if (newVal.length > 0) {
|
||||
defaultList.value = newVal;
|
||||
}
|
||||
},
|
||||
{ immediate: true }
|
||||
);
|
||||
watch(
|
||||
() => props.isPlaying,
|
||||
newVal => {
|
||||
if (newVal) {
|
||||
// 绿色
|
||||
bgColor.value = '#4dc100';
|
||||
} else {
|
||||
// 蓝色
|
||||
bgColor.value = '#4c5cf8';
|
||||
}
|
||||
}
|
||||
);
|
||||
function drawBars() {
|
||||
const bars = document.querySelectorAll('.bar');
|
||||
if (bars.length === 0) {
|
||||
cancelAnimationFrame(animationFrameId.value);
|
||||
return;
|
||||
}
|
||||
|
||||
const maxHeight = document.querySelector('.bars').clientHeight; // 最大高度为容器的高度
|
||||
|
||||
const averageVolume = props.dataArray.reduce((sum, value) => sum + value, 0) / props.dataArray.length;
|
||||
const normalizedVolume = props.isPlaying ? Math.random() : averageVolume / 128; // 将音量数据归一化为0到1之间
|
||||
|
||||
bars.forEach((bar, index) => {
|
||||
const minHeight = defaultList.value[index];
|
||||
const randomFactor = Math.random() * 1.5 + 0.5; // 随机因子
|
||||
const newHeight = Math.min(
|
||||
maxHeight,
|
||||
minHeight + (maxHeight - minHeight) * normalizedVolume * randomFactor
|
||||
); // 根据音量设置高度
|
||||
bar.style.height = `${newHeight}px`; // 设置新的高度
|
||||
bar.style.backgroundColor = bgColor.value;
|
||||
});
|
||||
|
||||
animationFrameId.value = requestAnimationFrame(drawBars);
|
||||
}
|
||||
const stopDraw = () => {
|
||||
if (animationFrameId.value) {
|
||||
cancelAnimationFrame(animationFrameId.value);
|
||||
}
|
||||
};
|
||||
</script>
|
||||
|
||||
<style lang="less" scoped>
|
||||
.bars {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
}
|
||||
.bar {
|
||||
// width: 6px;
|
||||
// margin: 0 2px;
|
||||
background-color: #4c5cf8;
|
||||
transition:
|
||||
height 0.1s,
|
||||
background-color 0.1s;
|
||||
border-radius: 5px; /* 圆角 */
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,8 @@
|
||||
/**
|
||||
* Configure and register global directives
|
||||
*/
|
||||
import ElTableInfiniteScroll from 'el-table-infinite-scroll';
|
||||
|
||||
export function setupGlobDirectives(app) {
|
||||
app.use(ElTableInfiniteScroll);
|
||||
}
|
||||
18
web_demos/minicpm-o_2.6/web_server/src/enums/index.js
Normal file
18
web_demos/minicpm-o_2.6/web_server/src/enums/index.js
Normal file
@@ -0,0 +1,18 @@
|
||||
export const voiceIdeasList = ['TBD', 'TBD', 'TBD'];
|
||||
export const videoIdeasList = ['TBD', 'TBD', 'TBD'];
|
||||
export const limitTime = 10 * 60; // 限制单次使用时常不超过10分钟
|
||||
export const tipsRemainingTime = 30; // 剩余30s时提醒用户
|
||||
// 初始音频波形
|
||||
export const voiceConfigList = [
|
||||
16, 16, 16, 16, 36, 58, 50, 70, 50, 58, 36, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 46, 28,
|
||||
60, 28, 68, 60, 28, 46, 16, 16, 16, 16, 16, 16, 16, 16, 36, 58, 50, 70, 50, 58, 36, 16, 16, 16, 16, 16, 16, 16, 16,
|
||||
16, 16, 16, 16, 16, 16, 16, 16, 46, 28, 60, 28, 68, 60, 28, 46, 16, 16, 16, 16
|
||||
];
|
||||
// 初始视频中的音频波形
|
||||
export const videoConfigList = [
|
||||
8, 8, 8, 8, 18, 28, 26, 36, 26, 28, 18, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 24, 14, 30, 14, 34, 30, 14,
|
||||
24, 8, 8, 8, 8, 8, 8, 8, 8, 18, 28, 26, 36, 26, 28, 18, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 24, 14, 30,
|
||||
14, 34, 30, 14, 24, 8, 8, 8, 8, 8, 8, 8, 8, 18, 28, 26, 36, 26, 28, 18, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
||||
8, 24, 14, 30, 14, 34, 30, 14, 24, 8, 8, 8, 8
|
||||
];
|
||||
export const showIdeasList = false;
|
||||
61
web_demos/minicpm-o_2.6/web_server/src/hooks/useHttp.js
Normal file
61
web_demos/minicpm-o_2.6/web_server/src/hooks/useHttp.js
Normal file
@@ -0,0 +1,61 @@
|
||||
import axios from 'axios';
|
||||
import { setNewUserId, getNewUserId } from './useRandomId';
|
||||
|
||||
// 创建实例时配置默认值
|
||||
const service = axios.create({
|
||||
baseURL: '/',
|
||||
timeout: 30000,
|
||||
responseType: 'json'
|
||||
});
|
||||
|
||||
// 请求拦截器
|
||||
service.interceptors.request.use(config => {
|
||||
if (config.url.includes('stream')) {
|
||||
config.timeout = 3000;
|
||||
}
|
||||
if (window.location.search) {
|
||||
config.url += window.location.search;
|
||||
}
|
||||
Object.assign(config.headers, ajaxHeader());
|
||||
return config;
|
||||
});
|
||||
|
||||
// 响应拦截器
|
||||
service.interceptors.response.use(
|
||||
response => {
|
||||
let res = response.data;
|
||||
if (response?.status === 200) {
|
||||
return Promise.resolve({
|
||||
code: 0,
|
||||
message: '',
|
||||
data: res
|
||||
});
|
||||
}
|
||||
return Promise.resolve({ code: -1, message: '网络异常，请稍后再试', data: null });
|
||||
},
|
||||
error => {
|
||||
const res = { code: -1, message: error?.response?.data?.detail || '网络异常，请稍后再试', data: null };
|
||||
return Promise.resolve(res);
|
||||
}
|
||||
);
|
||||
|
||||
export const ajaxHeader = () => {
|
||||
if (!localStorage.getItem('uid')) {
|
||||
setNewUserId();
|
||||
}
|
||||
return {
|
||||
'Content-Type': 'application/json;charset=UTF-8',
|
||||
Accept: 'application/json',
|
||||
service: 'minicpmo-server',
|
||||
uid: getNewUserId()
|
||||
};
|
||||
};
|
||||
|
||||
export default {
|
||||
get(url, params, config = {}) {
|
||||
return service.get(url, { params, ...config });
|
||||
},
|
||||
post(url, data, config = {}) {
|
||||
return service.post(url, data, { ...config });
|
||||
}
|
||||
};
|
||||
95
web_demos/minicpm-o_2.6/web_server/src/hooks/useQueue.js
Normal file
95
web_demos/minicpm-o_2.6/web_server/src/hooks/useQueue.js
Normal file
@@ -0,0 +1,95 @@
|
||||
export class TaskQueue {
|
||||
constructor() {
|
||||
this.tasks = [];
|
||||
this.isRunning = false;
|
||||
this.isPaused = false;
|
||||
this.currentTask = null;
|
||||
}
|
||||
|
||||
// 添加任务到队列
|
||||
addTask(task) {
|
||||
this.tasks.push(task);
|
||||
if (!this.isRunning) {
|
||||
this.start();
|
||||
}
|
||||
}
|
||||
|
||||
// 删除任务
|
||||
removeTask(taskToRemove) {
|
||||
this.tasks = this.tasks.filter(task => task !== taskToRemove);
|
||||
}
|
||||
|
||||
// 清空任务队列
|
||||
clearQueue() {
|
||||
this.tasks = [];
|
||||
}
|
||||
|
||||
// 暂停任务执行
|
||||
pause() {
|
||||
this.isPaused = true;
|
||||
}
|
||||
|
||||
// 恢复任务执行
|
||||
resume() {
|
||||
if (this.isPaused) {
|
||||
this.isPaused = false;
|
||||
if (!this.isRunning) {
|
||||
this.start();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 内部启动方法
|
||||
async start() {
|
||||
this.isRunning = true;
|
||||
while (this.tasks.length > 0 && !this.isPaused) {
|
||||
this.currentTask = this.tasks.shift();
|
||||
await this.currentTask();
|
||||
|
||||
// 检查是否暂停或任务队列已清空
|
||||
if (this.isPaused || this.tasks.length === 0) {
|
||||
this.isRunning = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
this.isRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
// 示例任务函数
|
||||
function exampleTask(id) {
|
||||
return () =>
|
||||
new Promise(resolve => {
|
||||
console.log(`Executing task ${id}`);
|
||||
setTimeout(() => {
|
||||
console.log(`Task ${id} completed`);
|
||||
resolve();
|
||||
}, 1000); // 每个任务耗时1秒
|
||||
});
|
||||
}
|
||||
|
||||
// 测试示例
|
||||
const queue = new TaskQueue();
|
||||
|
||||
// 添加任务到队列
|
||||
for (let i = 1; i <= 5; i++) {
|
||||
queue.addTask(exampleTask(i));
|
||||
}
|
||||
|
||||
// 暂停队列，在2.5秒后执行
|
||||
setTimeout(() => {
|
||||
console.log('Pausing queue...');
|
||||
queue.pause();
|
||||
}, 2500);
|
||||
|
||||
// 恢复队列，在4.5秒后执行
|
||||
setTimeout(() => {
|
||||
console.log('Resuming queue...');
|
||||
queue.resume();
|
||||
}, 4500);
|
||||
|
||||
// 清空队列，在3秒后执行
|
||||
setTimeout(() => {
|
||||
console.log('Clearing queue...');
|
||||
queue.clearQueue();
|
||||
}, 3000);
|
||||
@@ -0,0 +1,9 @@
|
||||
const uid = 'uid';
|
||||
export const setNewUserId = () => {
|
||||
const randomId = Math.random().toString(36).slice(2).toUpperCase();
|
||||
localStorage.setItem(uid, randomId);
|
||||
return randomId;
|
||||
};
|
||||
export const getNewUserId = () => {
|
||||
return localStorage.getItem('uid');
|
||||
};
|
||||
38
web_demos/minicpm-o_2.6/web_server/src/hooks/useVoice.js
Normal file
38
web_demos/minicpm-o_2.6/web_server/src/hooks/useVoice.js
Normal file
@@ -0,0 +1,38 @@
|
||||
const writeString = (view, offset, string) => {
|
||||
for (let i = 0; i < string.length; i++) {
|
||||
view.setUint8(offset + i, string.charCodeAt(i));
|
||||
}
|
||||
};
|
||||
const floatTo16BitPCM = (output, offset, input) => {
|
||||
for (let i = 0; i < input.length; i++, offset += 2) {
|
||||
const s = Math.max(-1, Math.min(1, input[i]));
|
||||
output.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
|
||||
}
|
||||
};
|
||||
// audio buffer to wav file, need add 44 length header
|
||||
export const encodeWAV = (samples, sampleRate) => {
|
||||
const buffer = new ArrayBuffer(44 + samples.length * 2);
|
||||
const view = new DataView(buffer);
|
||||
const numChannels = 1;
|
||||
const bitsPerSample = 16;
|
||||
|
||||
/* WAV 标头 */
|
||||
writeString(view, 0, 'RIFF');
|
||||
view.setUint32(4, 36 + samples.length * 2, true);
|
||||
writeString(view, 8, 'WAVE');
|
||||
writeString(view, 12, 'fmt ');
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true);
|
||||
view.setUint16(22, numChannels, true);
|
||||
view.setUint32(24, sampleRate, true);
|
||||
view.setUint32(28, (sampleRate * numChannels * bitsPerSample) / 8, true);
|
||||
view.setUint16(32, (numChannels * bitsPerSample) / 8, true);
|
||||
view.setUint16(34, bitsPerSample, true);
|
||||
writeString(view, 36, 'data');
|
||||
view.setUint32(40, samples.length * 2, true);
|
||||
|
||||
/* PCM 数据 */
|
||||
floatTo16BitPCM(view, 44, samples);
|
||||
|
||||
return new Blob([view], { type: 'audio/wav' });
|
||||
};
|
||||
36
web_demos/minicpm-o_2.6/web_server/src/i18n/en.json
Normal file
36
web_demos/minicpm-o_2.6/web_server/src/i18n/en.json
Normal file
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"menuTabVideo": "Realtime Video Call",
|
||||
"menuTabAudio": "Realtime Voice Call",
|
||||
"menuTabChatbot": "Chatbot",
|
||||
"videoCallBtn": "Call MiniCPM-omni",
|
||||
"audioCallBtn": "Call MiniCPM-omni",
|
||||
"hangUpBtn": "Hang Up",
|
||||
"notReadyBtn": "Not ready yet, please wait",
|
||||
"skipMessageBtn": "Skip this message",
|
||||
"feedbackDialogTitle": "Feedback issue",
|
||||
"modelConfigTitle": "Model Config",
|
||||
"audioInterruptionBtn": "Speech Interruption",
|
||||
"audioInterruptionTips": "When the \"voice interruption\" mode is enabled, it allows users to interrupt the model while it is speaking. The model will immediately terminate the previous round of generation and respond to the user's latest question.",
|
||||
"yes": "Yes",
|
||||
"no": "No",
|
||||
"videoQualityBtn": "HD Mode",
|
||||
"videoQualityTips": "When the \"high resulation\" mode is enabled, the model will perform high resolution encoding on the last frame, allowing the model to see more detailed parts.",
|
||||
"high": "High",
|
||||
"low": "Low",
|
||||
"vadThresholdBtn": "VAD Threshold",
|
||||
"vadThresholdTips": "The VAD threshold indicates how long the sound needs to be silent before triggering inference. If the VAD threshold is too low, it may trigger accidentally during speech pauses, while if it's too high, it will result in slower initial response.",
|
||||
"assistantPromptBtn": "Task Prompt",
|
||||
"assistantPromptTips": "Model task instructions are used to support different task objectives.",
|
||||
"useVoicePromptBtn": "Tone Color Prompt",
|
||||
"voiceClonePromptInput": "Tone Color Prompt",
|
||||
"voiceClonePromptTips": "Tone Color Prompt tips",
|
||||
"audioChoiceBtn": "Audio Choice",
|
||||
"defaultAudioBtn": "Default Audio",
|
||||
"customizationBtn": "Customization: Upload Audio",
|
||||
"toneColorOptions": "Voice Options",
|
||||
"toneColorOptionsTips": "We have provided a selection of sample tone colors, and you also have the option to choose \"none\" and instruct the model to create a new tone color.",
|
||||
"nullOption": "Null",
|
||||
"defaultOption": "Female 1(Default)",
|
||||
"femaleOption": "Female 2",
|
||||
"maleOption": "Male 1"
|
||||
}
|
||||
36
web_demos/minicpm-o_2.6/web_server/src/i18n/zh.json
Normal file
36
web_demos/minicpm-o_2.6/web_server/src/i18n/zh.json
Normal file
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"menuTabVideo": "实时视频通话",
|
||||
"menuTabAudio": "实时语音通话",
|
||||
"menuTabChatbot": "聊天机器人",
|
||||
"videoCallBtn": "视频通话",
|
||||
"audioCallBtn": "语音通话",
|
||||
"hangUpBtn": "挂断",
|
||||
"notReadyBtn": "服务繁忙，请稍后",
|
||||
"skipMessageBtn": "跳过当前对话",
|
||||
"feedbackDialogTitle": "请输入反馈意见",
|
||||
"modelConfigTitle": "模型配置",
|
||||
"audioInterruptionBtn": "语音打断",
|
||||
"audioInterruptionTips": "开启\"语音打断\"功能，支持在模型说话时打断模型，模型会立刻结束上一轮的生成，并支持用户最新的问题。",
|
||||
"yes": "是",
|
||||
"no": "否",
|
||||
"videoQualityBtn": "高清模式",
|
||||
"videoQualityTips": "开启高清模式，模型会在最后一帧对图片进行高清编码，可以使得模型看得清更细节的部分。",
|
||||
"high": "高清",
|
||||
"low": "低清",
|
||||
"vadThresholdBtn": "VAD阈值",
|
||||
"vadThresholdTips": "vad阈值表示声音静音多久才开始触发推理，vad阈值过低会导致说话气口误触，过高会导致首响更慢。",
|
||||
"assistantPromptBtn": "任务指令",
|
||||
"assistantPromptTips": "模型的任务指令，用于支持不同的任务目标",
|
||||
"useVoicePromptBtn": "音色指令",
|
||||
"voiceClonePromptInput": "音色指令",
|
||||
"voiceClonePromptTips": "我们的模型具有端到端的音色克隆能力，提供一段 5-7 秒的音频，模型在一定程度上可以用这种音色来说话。但基于法律考虑，我们的demo并不开启这个能力的试用。社区可以参照我们的开源代码自行适配。",
|
||||
"audioChoiceBtn": "音色选择",
|
||||
"defaultAudioBtn": "默认音色",
|
||||
"customizationBtn": "自定义：上传音频",
|
||||
"toneColorOptions": "语音选项",
|
||||
"toneColorOptionsTips": "我们提供了一些示例音色，也可以选择“无”并通过指令让模型创建音色。",
|
||||
"nullOption": "无",
|
||||
"defaultOption": "女一号(默认)",
|
||||
"femaleOption": "女二号",
|
||||
"maleOption": "男一号"
|
||||
}
|
||||
40
web_demos/minicpm-o_2.6/web_server/src/main.js
Normal file
40
web_demos/minicpm-o_2.6/web_server/src/main.js
Normal file
@@ -0,0 +1,40 @@
|
||||
import './styles/main.css';
|
||||
|
||||
import { router, setupRouter } from '@/router';
|
||||
import { setupRouterGuard } from '@/router/guard';
|
||||
import SvgIcon from '@/components/SvgIcon/index.vue';
|
||||
import { createI18n } from 'vue-i18n';
|
||||
|
||||
import App from './App.vue';
|
||||
import en from './i18n/en.json';
|
||||
import zh from './i18n/zh.json';
|
||||
|
||||
const savedLanguage = localStorage.getItem('language') || 'zh';
|
||||
|
||||
const i18n = createI18n({
|
||||
locale: savedLanguage, // 默认语言
|
||||
messages: {
|
||||
en,
|
||||
zh
|
||||
}
|
||||
});
|
||||
|
||||
const app = createApp(App);
|
||||
|
||||
// Configure routing
|
||||
// 配置路由
|
||||
setupRouter(app);
|
||||
|
||||
// router-guard
|
||||
// 路由守卫
|
||||
setupRouterGuard(router);
|
||||
|
||||
// Register global directive
|
||||
// 注册全局指令
|
||||
// setupGlobDirectives(app);
|
||||
|
||||
app.component('SvgIcon', SvgIcon);
|
||||
|
||||
app.use(i18n);
|
||||
|
||||
app.mount('#app');
|
||||
@@ -0,0 +1,5 @@
|
||||
import { createStateGuard } from './stateGuard';
|
||||
|
||||
export function setupRouterGuard(router) {
|
||||
createStateGuard(router);
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export function createStateGuard() {}
|
||||
16
web_demos/minicpm-o_2.6/web_server/src/router/index.js
Normal file
16
web_demos/minicpm-o_2.6/web_server/src/router/index.js
Normal file
@@ -0,0 +1,16 @@
|
||||
import { createRouter, createWebHistory } from 'vue-router';
|
||||
import { basicRoutes } from './menu';
|
||||
|
||||
// 创建一个可以被 Vue 应用程序使用的路由实例
|
||||
export const router = createRouter({
|
||||
// 创建一个 hash 历史记录。
|
||||
history: createWebHistory(import.meta.env.BASE_URL),
|
||||
// 路由列表。
|
||||
routes: basicRoutes
|
||||
});
|
||||
|
||||
// config router
|
||||
// 配置路由器
|
||||
export function setupRouter(app) {
|
||||
app.use(router);
|
||||
}
|
||||
10
web_demos/minicpm-o_2.6/web_server/src/router/menu/index.js
Normal file
10
web_demos/minicpm-o_2.6/web_server/src/router/menu/index.js
Normal file
@@ -0,0 +1,10 @@
|
||||
export const basicRoutes = [
|
||||
{
|
||||
path: '/',
|
||||
component: () => import('@/views/home/index.vue')
|
||||
},
|
||||
{
|
||||
path: '/:port',
|
||||
component: () => import('@/views/home/index.vue')
|
||||
}
|
||||
];
|
||||
56
web_demos/minicpm-o_2.6/web_server/src/styles/base.css
Normal file
56
web_demos/minicpm-o_2.6/web_server/src/styles/base.css
Normal file
@@ -0,0 +1,56 @@
|
||||
*,
|
||||
*::before,
|
||||
*::after {
|
||||
box-sizing: border-box;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar {
|
||||
width: 6px;
|
||||
height: 6px;
|
||||
}
|
||||
::-webkit-scrollbar-thumb {
|
||||
background: #e0e4ee;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
html,
|
||||
body {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
font-family:
|
||||
Inter,
|
||||
-apple-system,
|
||||
BlinkMacSystemFont,
|
||||
Segoe UI,
|
||||
SF Pro SC,
|
||||
SF Pro Display,
|
||||
SF Pro Icons,
|
||||
PingFang SC,
|
||||
Hiragino Sans GB,
|
||||
Microsoft YaHei,
|
||||
Helvetica Neue,
|
||||
Helvetica,
|
||||
Arial,
|
||||
sans-serif !important;
|
||||
background: #f3f3f3;
|
||||
transition:
|
||||
color 0.5s,
|
||||
background-color 0.5s;
|
||||
line-height: 1.3;
|
||||
font-size: 14px;
|
||||
font-weight: 400;
|
||||
color: var(--el-text-color-regular);
|
||||
text-rendering: optimizeLegibility;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
-moz-osx-font-smoothing: grayscale;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
#app {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
padding: 16px 4vw;
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
@import url('./variable.less');
|
||||
|
||||
.el-message {
|
||||
box-shadow: 0px 4px 13px 2px rgba(75, 79, 88, 0.11);
|
||||
border: none;
|
||||
border-radius: 8px;
|
||||
top: 60px !important;
|
||||
&--success,
|
||||
&--error {
|
||||
.el-message__content {
|
||||
color: rgb(10, 10, 10);
|
||||
font-size: 14px;
|
||||
}
|
||||
}
|
||||
.el-message-icon--error {
|
||||
color: var(--el-color-danger);
|
||||
font-size: 16px;
|
||||
}
|
||||
.el-message-icon--success {
|
||||
color: var(--el-color-success);
|
||||
font-size: 16px;
|
||||
}
|
||||
}
|
||||
.el-message.time-warning,
|
||||
.el-message.system-error {
|
||||
width: calc(100vw - 200px);
|
||||
padding: 16px 12px;
|
||||
border-radius: 12px;
|
||||
}
|
||||
.el-message.el-message--warning.time-warning {
|
||||
border: 1px solid #f9ac2a;
|
||||
background: #fef7ea;
|
||||
.el-icon {
|
||||
display: none;
|
||||
}
|
||||
.el-message__content {
|
||||
color: #2f333e;
|
||||
font-family: PingFang SC;
|
||||
font-size: 14px;
|
||||
font-style: normal;
|
||||
font-weight: 400;
|
||||
line-height: normal;
|
||||
padding-left: 28px;
|
||||
position: relative;
|
||||
}
|
||||
.el-message__content::before {
|
||||
position: absolute;
|
||||
content: '';
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
background: url('@/assets/svg/warning.svg') no-repeat;
|
||||
left: 0;
|
||||
}
|
||||
}
|
||||
.el-message.el-message--error.system-error {
|
||||
border: 1px solid #e72b00;
|
||||
background: #ffebe7;
|
||||
.el-icon {
|
||||
display: none;
|
||||
}
|
||||
.el-message__content {
|
||||
color: #2f333e;
|
||||
font-family: PingFang SC;
|
||||
font-size: 14px;
|
||||
font-style: normal;
|
||||
font-weight: 400;
|
||||
line-height: normal;
|
||||
padding-left: 28px;
|
||||
position: relative;
|
||||
}
|
||||
.el-message__content::before {
|
||||
position: absolute;
|
||||
content: '';
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
background: url('@/assets/svg/error.svg') no-repeat;
|
||||
left: 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
:root {
|
||||
--el-component-size-large: 48px;
|
||||
--el-component-size: 40px;
|
||||
--el-color-primary: #7661ff;
|
||||
--el-color-danger: #de0000;
|
||||
--el-color-warning: #ff7d00;
|
||||
--el-color-success: #00b42a;
|
||||
--el-text-color-regular: #0a0a0a;
|
||||
}
|
||||
.el-button {
|
||||
--el-button-height: var(--el-component-size);
|
||||
height: var(--el-button-height);
|
||||
&--large {
|
||||
--el-button-height: var(--el-component-size-large);
|
||||
height: var(--el-button-height);
|
||||
}
|
||||
&--primary {
|
||||
--el-button-bg-color: #7661ff;
|
||||
--el-button-text-color: var(--el-color-white);
|
||||
--el-button-border-color: #7661ff;
|
||||
--el-button-hover-bg-color: rgb(159, 144, 255);
|
||||
--el-button-hover-text-color: var(--el-color-white);
|
||||
--el-button-hover-border-color: rgb(159, 144, 255);
|
||||
--el-button-active-bg-color: rgb(98, 82, 208);
|
||||
--el-button-active-border-color: rgb(98, 82, 208);
|
||||
--el-button-disabled-bg-color: #d4cdff;
|
||||
--el-button-disabled-border-color: #d4cdff;
|
||||
}
|
||||
}
|
||||
.el-checkbox {
|
||||
--el-checkbox-border-radius: 4px;
|
||||
--el-checkbox-input-border: 1px solid rgb(188, 188, 188);
|
||||
--el-checkbox-input-border-color-hover: rgb(61, 92, 255);
|
||||
--el-checkbox-checked-bg-color: rgb(61, 92, 255);
|
||||
--el-checkbox-checked-input-border-color: rgb(61, 92, 255);
|
||||
}
|
||||
.el-dialog {
|
||||
&__header {
|
||||
padding-bottom: 20px;
|
||||
}
|
||||
&__title {
|
||||
--el-text-color-primary: rgb(10, 10, 10);
|
||||
--el-dialog-title-font-size: 18px;
|
||||
--el-dialog-font-line-height: 20px;
|
||||
}
|
||||
}
|
||||
.el-message {
|
||||
--el-message-padding: 11px 20px;
|
||||
--el-message-bg-color: rgb(255, 255, 255);
|
||||
--el-message-text-color: #de0000;
|
||||
}
|
||||
30
web_demos/minicpm-o_2.6/web_server/src/styles/main.css
Normal file
30
web_demos/minicpm-o_2.6/web_server/src/styles/main.css
Normal file
@@ -0,0 +1,30 @@
|
||||
@import './base.css';
|
||||
@import './variable.css';
|
||||
|
||||
.layout-root {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.layout-main {
|
||||
flex: 1 1 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
padding: 0 var(--layout-main-padding);
|
||||
}
|
||||
|
||||
.layout-footer {
|
||||
width: 100%;
|
||||
max-width: var(--layout-content-width);
|
||||
height: fit-content;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
padding: 0 var(--layout-main-padding);
|
||||
margin: auto;
|
||||
}
|
||||
|
||||
:focus-visible {
|
||||
outline: none;
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
:root {
|
||||
--layout-sidebar-width: 56px;
|
||||
--layout-sidebar-left-space: 8px;
|
||||
--layout-main-padding: 8px;
|
||||
--layout-main-minWidth: calc(var(--layout-content-width) + var(--layout-main-padding) * 2);
|
||||
--layout-content-width: 780px;
|
||||
}
|
||||
44
web_demos/minicpm-o_2.6/web_server/src/utils/index.js
Normal file
44
web_demos/minicpm-o_2.6/web_server/src/utils/index.js
Normal file
@@ -0,0 +1,44 @@
|
||||
// 判断终端是pc还是移动端
|
||||
export const isMobile = () => {
|
||||
let flag = /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini|Linux/i.test(navigator.userAgent);
|
||||
const platform = navigator.platform;
|
||||
// iPad上的Safari
|
||||
if (platform === 'MacIntel' && navigator.maxTouchPoints > 1) {
|
||||
flag = true;
|
||||
}
|
||||
return flag;
|
||||
};
|
||||
// 单片语音长度(单位：ms)
|
||||
const voicePerLength = 200;
|
||||
|
||||
// 图片计数，算出在哪一次发送语音时，同时发送图片。例如一片语音100ms，一秒钟发送一次语音，即发送的第10片语音时需要带一张图片
|
||||
export const maxCount = 1000 / voicePerLength;
|
||||
|
||||
export const getChunkLength = sampleRate => {
|
||||
return sampleRate * (voicePerLength / 1000);
|
||||
};
|
||||
|
||||
export const isAvailablePort = port => {
|
||||
return [
|
||||
8000, 8001, 8002, 8003, 8004, 8010, 8011, 8012, 8013, 8014, 8020, 8021, 8022, 8023, 8024, 8025, 8026, 8027,
|
||||
8028, 32449
|
||||
].includes(port);
|
||||
};
|
||||
|
||||
// 文件转base64格式
|
||||
export const fileToBase64 = file => {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!file) {
|
||||
reject('文件不能为空');
|
||||
}
|
||||
const reader = new FileReader();
|
||||
reader.onload = e => {
|
||||
const base64String = e.target.result;
|
||||
resolve(base64String);
|
||||
};
|
||||
reader.onerror = () => {
|
||||
reject('文件转码失败');
|
||||
};
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
};
|
||||
91
web_demos/minicpm-o_2.6/web_server/src/utils/websocket.js
Normal file
91
web_demos/minicpm-o_2.6/web_server/src/utils/websocket.js
Normal file
@@ -0,0 +1,91 @@
|
||||
class WebSocketClient {
|
||||
constructor(url, maxReconnectAttempts = 5, reconnectInterval = 5000) {
|
||||
this.url = url;
|
||||
this.socket = null;
|
||||
this.eventHandlers = {};
|
||||
this.maxReconnectAttempts = maxReconnectAttempts;
|
||||
this.reconnectInterval = reconnectInterval;
|
||||
this.reconnectAttempts = 0;
|
||||
this.reconnectTimer = null;
|
||||
}
|
||||
|
||||
connect() {
|
||||
this.reconnectAttempts = 0;
|
||||
this.establishConnection();
|
||||
}
|
||||
|
||||
establishConnection() {
|
||||
this.socket = new WebSocket(this.url);
|
||||
|
||||
this.socket.onopen = () => {
|
||||
console.log('WebSocket connection opened');
|
||||
this.reconnectAttempts = 0; // Reset reconnect attempts on successful connection
|
||||
this.emit('open');
|
||||
};
|
||||
|
||||
this.socket.onclose = event => {
|
||||
console.log('WebSocket connection closed', event);
|
||||
this.emit('close', event);
|
||||
// 1005为主动关闭websocket
|
||||
if (event.code !== 1005) {
|
||||
this.reconnect();
|
||||
}
|
||||
};
|
||||
|
||||
this.socket.onerror = error => {
|
||||
console.error('WebSocket error', error);
|
||||
this.emit('error', error);
|
||||
// Optionally, you may want to trigger a reconnect on error as well
|
||||
// this.reconnect();
|
||||
};
|
||||
|
||||
this.socket.onmessage = message => {
|
||||
// console.log('WebSocket message received', message.data);
|
||||
this.emit('message', message.data);
|
||||
};
|
||||
}
|
||||
|
||||
send(data) {
|
||||
if (this.socket && this.socket.readyState === WebSocket.OPEN) {
|
||||
this.socket.send(data);
|
||||
} else {
|
||||
console.error('WebSocket is not open');
|
||||
}
|
||||
}
|
||||
|
||||
on(event, handler) {
|
||||
// if (!this.eventHandlers[event]) {
|
||||
this.eventHandlers[event] = [];
|
||||
// }
|
||||
this.eventHandlers[event].push(handler);
|
||||
// console.log('Event handler added:', this.eventHandlers, event);
|
||||
}
|
||||
|
||||
emit(event, ...args) {
|
||||
if (this.eventHandlers[event]) {
|
||||
this.eventHandlers[event].forEach(handler => handler(...args));
|
||||
}
|
||||
}
|
||||
|
||||
close() {
|
||||
if (this.socket) {
|
||||
this.socket.close();
|
||||
}
|
||||
clearTimeout(this.reconnectTimer);
|
||||
}
|
||||
|
||||
reconnect() {
|
||||
if (this.reconnectAttempts < this.maxReconnectAttempts) {
|
||||
console.log(`Reconnecting attempt ${this.reconnectAttempts + 1}/${this.maxReconnectAttempts}...`);
|
||||
this.reconnectTimer = setTimeout(() => {
|
||||
this.reconnectAttempts++;
|
||||
this.establishConnection();
|
||||
}, this.reconnectInterval);
|
||||
} else {
|
||||
console.error('Max reconnect attempts reached. WebSocket will not attempt to reconnect.');
|
||||
this.emit('max-reconnect-attempts');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default WebSocketClient;
|
||||
@@ -0,0 +1,3 @@
|
||||
<template>
|
||||
<div>Chatbot</div>
|
||||
</template>
|
||||
@@ -0,0 +1,971 @@
|
||||
<template>
|
||||
<!-- <ExtraInfo webVersion="非websocket_0111" :modelVersion="modelVersion" /> -->
|
||||
<div class="video-page">
|
||||
<div class="video-page-header">
|
||||
<div class="voice-container" v-if="!isCalling">
|
||||
<SvgIcon name="voice" class="voice-icon" />
|
||||
<SvgIcon name="voice" class="voice-icon" />
|
||||
<SvgIcon name="voice" class="voice-icon" />
|
||||
</div>
|
||||
<div class="voice-container" v-else>
|
||||
<Voice
|
||||
:dataArray="dataArray"
|
||||
:isCalling="isCalling"
|
||||
:isPlaying="playing"
|
||||
:configList="videoConfigList"
|
||||
:boxStyle="{ height: '45px' }"
|
||||
:itemStyle="{ width: '3px', margin: '0 1px' }"
|
||||
/>
|
||||
</div>
|
||||
<!-- <SelectTimbre v-model:timbre="timbre" v-model:audioData="audioData" v-model:disabled="isCalling" /> -->
|
||||
</div>
|
||||
<div class="video-page-content">
|
||||
<div class="video-page-content-video" v-loading="loading" element-loading-background="#f3f3f3">
|
||||
<video ref="videoRef" autoplay playsinline muted />
|
||||
<canvas ref="canvasRef" canvas-id="canvasId" style="display: none" />
|
||||
<div class="switch-camera" v-if="isMobile()" @click="switchCamera">
|
||||
<SvgIcon name="switch-camera" class="icon" />
|
||||
</div>
|
||||
</div>
|
||||
<div class="video-page-content-right">
|
||||
<div class="output-content">
|
||||
<ModelOutput
|
||||
v-if="outputData.length > 0"
|
||||
:outputData="outputData"
|
||||
containerClass="output-content"
|
||||
/>
|
||||
</div>
|
||||
<div class="skip-box">
|
||||
<!-- <DelayTips
|
||||
v-if="delayTimestamp > 200 || delayCount > 2"
|
||||
:delayTimestamp="delayTimestamp"
|
||||
:delayCount="delayCount"
|
||||
/> -->
|
||||
<LikeAndDislike v-model:feedbackStatus="feedbackStatus" v-model:curResponseId="curResponseId" />
|
||||
<SkipBtn :disabled="skipDisabled" @click="skipVoice" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="video-page-btn">
|
||||
<el-button v-show="!isCalling" type="success" :disabled="callDisabled" @click="initRecording">
|
||||
{{ callDisabled ? t('notReadyBtn') : t('videoCallBtn') }}
|
||||
</el-button>
|
||||
<el-button v-show="isCalling" @click="stopRecording" type="danger">
|
||||
<SvgIcon name="phone-icon" className="phone-icon" />
|
||||
<span class="btn-text">{{ t('hangUpBtn') }}</span>
|
||||
<CountDown v-model="isCalling" @timeUp="stopRecording" />
|
||||
</el-button>
|
||||
</div>
|
||||
<IdeasList v-if="showIdeasList" :ideasList="videoIdeasList" />
|
||||
</div>
|
||||
</template>
|
||||
<script setup>
|
||||
import { sendMessage, stopMessage, uploadConfig } from '@/apis';
|
||||
import { encodeWAV } from '@/hooks/useVoice';
|
||||
import { getNewUserId, setNewUserId } from '@/hooks/useRandomId';
|
||||
import { fetchEventSource } from '@microsoft/fetch-event-source';
|
||||
import { MicVAD } from '@ricky0123/vad-web';
|
||||
import { videoIdeasList, videoConfigList, showIdeasList } from '@/enums';
|
||||
import { isMobile, maxCount, getChunkLength } from '@/utils';
|
||||
import { mergeBase64ToBlob } from './merge';
|
||||
import { useI18n } from 'vue-i18n';
|
||||
|
||||
const { t } = useI18n();
|
||||
import WebSocketService from '@/utils/websocket';
|
||||
|
||||
let ctrl = new AbortController();
|
||||
let socket = null;
|
||||
const audioData = ref({
|
||||
base64Str: '',
|
||||
type: 'mp3'
|
||||
}); // 自定义音色base64
|
||||
const isCalling = defineModel();
|
||||
const videoRef = ref();
|
||||
const videoStream = ref(null);
|
||||
const interval = ref();
|
||||
const canvasRef = ref();
|
||||
const videoImage = ref([]);
|
||||
const videoLoaded = ref(false);
|
||||
const taskQueue = ref([]);
|
||||
const running = ref(false);
|
||||
const outputData = ref([]);
|
||||
const isFirstReturn = ref(true);
|
||||
const audioPlayQueue = ref([]);
|
||||
const base64List = ref([]);
|
||||
const playing = ref(false);
|
||||
const timbre = ref([1]);
|
||||
const isReturnError = ref(false);
|
||||
|
||||
const textQueue = ref('');
|
||||
const textAnimationInterval = ref();
|
||||
|
||||
const analyser = ref();
|
||||
const dataArray = ref();
|
||||
const animationFrameId = ref();
|
||||
const skipDisabled = ref(true);
|
||||
const stop = ref(false);
|
||||
const isFrontCamera = ref(true);
|
||||
const loading = ref(false);
|
||||
|
||||
const isEnd = ref(false); // sse接口关闭，认为模型已完成本次返回
|
||||
|
||||
const isFirstPiece = ref(true);
|
||||
const allVoice = ref([]);
|
||||
const callDisabled = ref(true);
|
||||
|
||||
const feedbackStatus = ref('');
|
||||
const curResponseId = ref('');
|
||||
const delayTimestamp = ref(0); // 当前发送片延时
|
||||
const delayCount = ref(0); // 当前剩余多少ms未发送到接口
|
||||
|
||||
const modelVersion = ref('');
|
||||
|
||||
let mediaStream;
|
||||
let audioRecorder;
|
||||
let audioStream;
|
||||
let intervalId;
|
||||
let audioContext;
|
||||
let audioChunks = [];
|
||||
let count = 0;
|
||||
let audioDOM;
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
stopRecording();
|
||||
});
|
||||
const vadStartTime = ref();
|
||||
let myvad = null;
|
||||
let vadTimer = null; // vad定时器，用于检测1s内人声是否停止，1s内停止，可认为是vad误触，直接忽略，1s内未停止，则认为是人声，已自动跳过当前对话
|
||||
const vadStart = async () => {
|
||||
myvad = await MicVAD.new({
|
||||
onSpeechStart: () => {
|
||||
console.log('Speech start', +new Date());
|
||||
// if (!skipDisabled.value) {
|
||||
vadTimer && clearTimeout(vadTimer);
|
||||
vadTimer = setTimeout(() => {
|
||||
// vadStartTime.value = +new Date();
|
||||
console.log('打断时间: ', +new Date());
|
||||
skipVoice();
|
||||
}, 500);
|
||||
// }
|
||||
},
|
||||
onSpeechEnd: audio => {
|
||||
vadTimer && clearTimeout(vadTimer);
|
||||
console.log('Speech end', +new Date());
|
||||
// debugger;
|
||||
// do something with `audio` (Float32Array of audio samples at sample rate 16000)...
|
||||
},
|
||||
baseAssetPath: '/'
|
||||
});
|
||||
myvad.start();
|
||||
};
|
||||
onMounted(async () => {
|
||||
const { code, message } = await stopMessage();
|
||||
if (code !== 0) {
|
||||
ElMessage({
|
||||
type: 'error',
|
||||
message: message,
|
||||
duration: 3000,
|
||||
customClass: 'system-error'
|
||||
});
|
||||
return;
|
||||
}
|
||||
callDisabled.value = false;
|
||||
});
|
||||
const delay = ms => {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
};
|
||||
const initRecording = async () => {
|
||||
uploadUserConfig()
|
||||
.then(async () => {
|
||||
if (!audioDOM) {
|
||||
audioDOM = new Audio();
|
||||
audioDOM.playsinline = true;
|
||||
audioDOM.preload = 'auto';
|
||||
}
|
||||
// 每次call都需要生成新uid
|
||||
setNewUserId();
|
||||
buildConnect();
|
||||
await delay(100);
|
||||
// if (socket) {
|
||||
// socket.close();
|
||||
// }
|
||||
// socket = new WebSocketService(
|
||||
// `/ws/stream${window.location.search}&uid=${getNewUserId()}&service=minicpmo-server`
|
||||
// );
|
||||
// socket.connect();
|
||||
|
||||
initVideoStream('environment');
|
||||
if (localStorage.getItem('canStopByVoice') === 'true') {
|
||||
console.log('vad start');
|
||||
vadStart();
|
||||
}
|
||||
})
|
||||
.catch(() => {});
|
||||
};
|
||||
// 切换摄像头
|
||||
const switchCamera = () => {
|
||||
if (!isCalling.value) {
|
||||
return;
|
||||
}
|
||||
isFrontCamera.value = !isFrontCamera.value;
|
||||
const facingMode = isFrontCamera.value ? 'environment' : 'user'; // 'user' 前置, 'environment' 后置
|
||||
initVideoStream(facingMode);
|
||||
};
|
||||
const initVideoStream = async facingMode => {
|
||||
if (mediaStream) {
|
||||
mediaStream.getTracks().forEach(track => track.stop());
|
||||
videoStream.value = null;
|
||||
}
|
||||
outputData.value = [];
|
||||
isCalling.value = true;
|
||||
loading.value = true;
|
||||
if (!videoStream.value) {
|
||||
try {
|
||||
mediaStream = await window.navigator.mediaDevices.getUserMedia({
|
||||
video: { facingMode },
|
||||
audio: true
|
||||
});
|
||||
videoStream.value = mediaStream;
|
||||
videoRef.value.srcObject = mediaStream;
|
||||
loading.value = false;
|
||||
console.log('打开后： ', +new Date());
|
||||
// takePhotos();
|
||||
audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 });
|
||||
console.log('samplate: ', audioContext);
|
||||
const audioSource = audioContext.createMediaStreamSource(mediaStream);
|
||||
interval.value = setInterval(() => dealImage(), 50);
|
||||
// 创建 ScriptProcessorNode 用于捕获音频数据
|
||||
const processor = audioContext.createScriptProcessor(256, 1, 1);
|
||||
|
||||
processor.onaudioprocess = event => {
|
||||
if (!isCalling.value) return;
|
||||
if (isReturnError.value) {
|
||||
stopRecording();
|
||||
return;
|
||||
}
|
||||
const data = event.inputBuffer.getChannelData(0);
|
||||
audioChunks.push(new Float32Array(data));
|
||||
// 检查是否已经收集到1秒钟的数据
|
||||
const totalBufferLength = audioChunks.reduce((total, curr) => total + curr.length, 0);
|
||||
const chunkLength = getChunkLength(audioContext.sampleRate);
|
||||
if (totalBufferLength >= chunkLength) {
|
||||
// 合并到一个完整的数据数组，并裁剪成1秒钟
|
||||
const mergedBuffer = mergeBuffers(audioChunks, totalBufferLength);
|
||||
const oneSecondBuffer = mergedBuffer.slice(0, audioContext.sampleRate);
|
||||
|
||||
// 保存并处理成WAV格式
|
||||
addQueue(+new Date(), () => saveAudioChunk(oneSecondBuffer, +new Date()));
|
||||
|
||||
// 保留多余的数据备用
|
||||
audioChunks = [mergedBuffer.slice(audioContext.sampleRate)];
|
||||
}
|
||||
};
|
||||
analyser.value = audioContext.createAnalyser();
|
||||
// 将音频节点连接到分析器
|
||||
audioSource.connect(analyser.value);
|
||||
// 分析器设置
|
||||
analyser.value.fftSize = 256;
|
||||
const bufferLength = analyser.value.frequencyBinCount;
|
||||
dataArray.value = new Uint8Array(bufferLength);
|
||||
// 开始绘制音波
|
||||
drawBars();
|
||||
|
||||
audioSource.connect(processor);
|
||||
processor.connect(audioContext.destination);
|
||||
} catch {}
|
||||
}
|
||||
};
|
||||
const drawText = async () => {
|
||||
if (textQueue.value.length > 0) {
|
||||
outputData.value[outputData.value.length - 1].text += textQueue.value[0];
|
||||
textQueue.value = textQueue.value.slice(1);
|
||||
} else {
|
||||
cancelAnimationFrame(textAnimationInterval.value);
|
||||
}
|
||||
textAnimationInterval.value = requestAnimationFrame(drawText);
|
||||
};
|
||||
const getStopValue = () => {
|
||||
return stop.value;
|
||||
};
|
||||
const getPlayingValue = () => {
|
||||
return playing.value;
|
||||
};
|
||||
const getStopStatus = () => {
|
||||
return localStorage.getItem('canStopByVoice') === 'true';
|
||||
};
|
||||
const saveAudioChunk = (buffer, timestamp) => {
|
||||
return new Promise(resolve => {
|
||||
if (!getStopStatus() && getPlayingValue()) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const wavBlob = encodeWAV(buffer, audioContext.sampleRate);
|
||||
let reader = new FileReader();
|
||||
reader.readAsDataURL(wavBlob);
|
||||
|
||||
reader.onloadend = async function () {
|
||||
let base64data = reader.result.split(',')[1];
|
||||
const imgBase64 = videoImage.value[videoImage.value.length - 1]?.src;
|
||||
if (!(base64data && imgBase64)) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const strBase64 = imgBase64.split(',')[1];
|
||||
count++;
|
||||
let obj = {
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'input_audio',
|
||||
input_audio: {
|
||||
data: base64data,
|
||||
format: 'wav',
|
||||
timestamp: String(timestamp)
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
};
|
||||
obj.messages[0].content.unshift({
|
||||
type: 'image_data',
|
||||
image_data: {
|
||||
data: count === maxCount ? strBase64 : '',
|
||||
type: 2
|
||||
}
|
||||
});
|
||||
if (count === maxCount) {
|
||||
count = 0;
|
||||
}
|
||||
// socket.send(JSON.stringify(obj));
|
||||
// socket.on('message', data => {
|
||||
// console.log('message: ', data);
|
||||
// delayTimestamp.value = +new Date() - timestamp;
|
||||
// delayCount.value = taskQueue.value.length;
|
||||
// resolve();
|
||||
// });
|
||||
// 将Base64音频数据发送到后端
|
||||
try {
|
||||
await sendMessage(obj);
|
||||
delayTimestamp.value = +new Date() - timestamp;
|
||||
delayCount.value = taskQueue.value.length;
|
||||
} catch (err) {}
|
||||
resolve();
|
||||
};
|
||||
});
|
||||
};
|
||||
const mergeBuffers = (buffers, length) => {
|
||||
const result = new Float32Array(length);
|
||||
let offset = 0;
|
||||
for (let buffer of buffers) {
|
||||
result.set(buffer, offset);
|
||||
offset += buffer.length;
|
||||
}
|
||||
return result;
|
||||
};
|
||||
const stopRecording = () => {
|
||||
isCalling.value = false;
|
||||
clearInterval(interval.value);
|
||||
interval.value = null;
|
||||
if (audioRecorder && audioRecorder.state !== 'inactive') {
|
||||
audioRecorder.stop();
|
||||
}
|
||||
if (animationFrameId.value) {
|
||||
cancelAnimationFrame(animationFrameId.value);
|
||||
}
|
||||
if (audioContext && audioContext.state !== 'closed') {
|
||||
audioContext.close();
|
||||
}
|
||||
destroyVideoStream();
|
||||
taskQueue.value = [];
|
||||
audioPlayQueue.value = [];
|
||||
base64List.value = [];
|
||||
ctrl.abort();
|
||||
ctrl = new AbortController();
|
||||
isReturnError.value = false;
|
||||
skipDisabled.value = true;
|
||||
playing.value = false;
|
||||
audioDOM?.pause();
|
||||
stopMessage();
|
||||
if (socket) {
|
||||
socket.close();
|
||||
}
|
||||
if (
|
||||
outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
|
||||
outputData.value[outputData.value.length - 1].audio === '' &&
|
||||
allVoice.value.length > 0
|
||||
) {
|
||||
outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
|
||||
}
|
||||
myvad && myvad.destroy();
|
||||
};
|
||||
// 建立连接
|
||||
const buildConnect = () => {
|
||||
const obj = {
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [{ type: 'none' }]
|
||||
}
|
||||
],
|
||||
stream: true
|
||||
};
|
||||
isEnd.value = false;
|
||||
ctrl.abort();
|
||||
ctrl = new AbortController();
|
||||
const url = `/api/v1/completions${window.location.search}`;
|
||||
|
||||
fetchEventSource(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
service: 'minicpmo-server',
|
||||
uid: getNewUserId()
|
||||
},
|
||||
body: JSON.stringify(obj),
|
||||
signal: ctrl.signal,
|
||||
openWhenHidden: true,
|
||||
async onopen(response) {
|
||||
isFirstPiece.value = true;
|
||||
isFirstReturn.value = true;
|
||||
allVoice.value = [];
|
||||
base64List.value = [];
|
||||
console.log('onopen', response);
|
||||
if (response.status !== 200) {
|
||||
ElMessage({
|
||||
type: 'error',
|
||||
message: 'At limit. Please try again soon.',
|
||||
duration: 3000,
|
||||
customClass: 'system-error'
|
||||
});
|
||||
isReturnError.value = true;
|
||||
} else {
|
||||
isReturnError.value = false;
|
||||
drawText();
|
||||
}
|
||||
},
|
||||
onmessage(msg) {
|
||||
const data = JSON.parse(msg.data);
|
||||
if (data.response_id) {
|
||||
curResponseId.value = data.response_id;
|
||||
}
|
||||
if (data.choices[0]?.text) {
|
||||
textQueue.value += data.choices[0].text.replace('<end>', '');
|
||||
console.warn('text return time -------------------------------', +new Date());
|
||||
}
|
||||
// 首次返回的是前端发给后端的音频片段，需要单独处理
|
||||
if (isFirstReturn.value) {
|
||||
console.log('第一次');
|
||||
isFirstReturn.value = false;
|
||||
// 如果后端返回的音频为空，需要重连
|
||||
if (!data.choices[0].audio) {
|
||||
buildConnect();
|
||||
return;
|
||||
}
|
||||
outputData.value.push({
|
||||
type: 'USER',
|
||||
audio: `data:audio/wav;base64,${data.choices[0].audio}`
|
||||
});
|
||||
outputData.value.push({
|
||||
type: 'BOT',
|
||||
text: '',
|
||||
audio: ''
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (data.choices[0]?.audio) {
|
||||
console.log('audio return time -------------------------------', +new Date());
|
||||
if (!getStopValue() && isCalling.value) {
|
||||
skipDisabled.value = false;
|
||||
base64List.value.push(`data:audio/wav;base64,${data.choices[0].audio}`);
|
||||
addAudioQueue(() => truePlay(data.choices[0].audio));
|
||||
}
|
||||
allVoice.value.push(`data:audio/wav;base64,${data.choices[0].audio}`);
|
||||
} else {
|
||||
// 发生异常了，直接重连
|
||||
buildConnect();
|
||||
}
|
||||
if (data.choices[0].text.includes('<end>')) {
|
||||
// isEnd.value = true;
|
||||
console.log('收到结束标记了:', +new Date());
|
||||
if (
|
||||
outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
|
||||
outputData.value[outputData.value.length - 1].audio === '' &&
|
||||
allVoice.value.length > 0
|
||||
) {
|
||||
outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
|
||||
}
|
||||
}
|
||||
},
|
||||
onclose() {
|
||||
console.log('onclose', +new Date());
|
||||
isEnd.value = true;
|
||||
if (
|
||||
outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
|
||||
outputData.value[outputData.value.length - 1].audio === '' &&
|
||||
allVoice.value.length > 0
|
||||
) {
|
||||
outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
|
||||
}
|
||||
// sse关闭后，如果待播放的音频列表为空，说明模型出错了，此次连接没有返回音频，则直接重连
|
||||
vadStartTime.value = +new Date();
|
||||
if (audioPlayQueue.value.length === 0) {
|
||||
let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
|
||||
if (startIndex !== -1) {
|
||||
taskQueue.value = taskQueue.value.slice(startIndex);
|
||||
}
|
||||
buildConnect();
|
||||
}
|
||||
},
|
||||
onerror(err) {
|
||||
console.log('onerror', err);
|
||||
ctrl.abort();
|
||||
ctrl = new AbortController();
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
};
|
||||
// 返回的语音放到队列里，挨个播放
|
||||
const addAudioQueue = async item => {
|
||||
audioPlayQueue.value.push(item);
|
||||
if (isFirstPiece.value) {
|
||||
await delay(1500);
|
||||
isFirstPiece.value = false;
|
||||
}
|
||||
if (audioPlayQueue.value.length > 0 && !playing.value) {
|
||||
playing.value = true;
|
||||
playAudio();
|
||||
}
|
||||
};
|
||||
// 控制播放队列执行
|
||||
const playAudio = () => {
|
||||
console.log('剩余播放列表:', audioPlayQueue.value, +new Date());
|
||||
|
||||
if (!isEnd.value && base64List.value.length >= 2) {
|
||||
const remainLen = base64List.value.length;
|
||||
const blob = mergeBase64ToBlob(base64List.value);
|
||||
audioDOM.src = blob;
|
||||
audioDOM.play();
|
||||
console.error('前期合并后播放开始时间: ', +new Date());
|
||||
audioDOM.onended = () => {
|
||||
console.error('前期合并后播放结束时间: ', +new Date());
|
||||
base64List.value = base64List.value.slice(remainLen);
|
||||
audioPlayQueue.value = audioPlayQueue.value.slice(remainLen);
|
||||
playAudio();
|
||||
};
|
||||
return;
|
||||
}
|
||||
if (isEnd.value && base64List.value.length >= 2) {
|
||||
const blob = mergeBase64ToBlob(base64List.value);
|
||||
audioDOM.src = blob;
|
||||
audioDOM.play();
|
||||
console.error('合并后播放开始时间: ', +new Date());
|
||||
audioDOM.onended = () => {
|
||||
console.error('合并后播放结束时间: ', +new Date());
|
||||
// URL.revokeObjectURL(url);
|
||||
base64List.value = [];
|
||||
audioPlayQueue.value = [];
|
||||
playing.value = false;
|
||||
skipDisabled.value = true;
|
||||
if (isCalling.value && !isReturnError.value) {
|
||||
// skipDisabled.value = true;
|
||||
taskQueue.value = [];
|
||||
// 打断前记录一下打断时间或vad触发事件
|
||||
// vadStartTime.value = +new Date();
|
||||
// // 每次完成后只保留当前时刻往前推1s的语音
|
||||
// console.log(
|
||||
// '截取前长度:',
|
||||
// taskQueue.value.map(item => item.time)
|
||||
// );
|
||||
// let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
|
||||
// if (startIndex !== -1) {
|
||||
// taskQueue.value = taskQueue.value.slice(startIndex);
|
||||
// console.log(
|
||||
// '截取后长度:',
|
||||
// taskQueue.value.map(item => item.time),
|
||||
// vadStartTime.value
|
||||
// );
|
||||
// }
|
||||
buildConnect();
|
||||
}
|
||||
};
|
||||
return;
|
||||
}
|
||||
base64List.value.shift();
|
||||
const _truePlay = audioPlayQueue.value.shift();
|
||||
if (_truePlay) {
|
||||
_truePlay().finally(() => {
|
||||
playAudio();
|
||||
});
|
||||
} else {
|
||||
playing.value = false;
|
||||
if (isEnd.value) {
|
||||
console.warn('play done................');
|
||||
skipDisabled.value = true;
|
||||
}
|
||||
// 播放完成后且正在通话且接口未返回错误时开始下一次连接
|
||||
if (isEnd.value && isCalling.value && !isReturnError.value) {
|
||||
// skipDisabled.value = true;
|
||||
taskQueue.value = [];
|
||||
// 跳过之后，只保留当前时间点两秒内到之后的音频片段
|
||||
// vadStartTime.value = +new Date();
|
||||
// console.log(
|
||||
// '截取前长度:',
|
||||
// taskQueue.value.map(item => item.time)
|
||||
// );
|
||||
// let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
|
||||
// if (startIndex !== -1) {
|
||||
// taskQueue.value = taskQueue.value.slice(startIndex);
|
||||
// console.log(
|
||||
// '截取后长度:',
|
||||
// taskQueue.value.map(item => item.time),
|
||||
// vadStartTime.value
|
||||
// );
|
||||
// }
|
||||
buildConnect();
|
||||
}
|
||||
}
|
||||
};
|
||||
// 播放音频
|
||||
const truePlay = voice => {
|
||||
console.log('promise: ', +new Date());
|
||||
return new Promise(resolve => {
|
||||
audioDOM.src = 'data:audio/wav;base64,' + voice;
|
||||
console.error('播放开始时间:', +new Date());
|
||||
audioDOM
|
||||
.play()
|
||||
.then(() => {
|
||||
console.log('Audio played successfully');
|
||||
})
|
||||
.catch(error => {
|
||||
if (error.name === 'NotAllowedError' || error.name === 'SecurityError') {
|
||||
console.error('User interaction required or permission issue:', error);
|
||||
// ElMessage.warning('音频播放失败');
|
||||
console.error('播放失败时间');
|
||||
// alert('Please interact with the page (like clicking a button) to enable audio playback.');
|
||||
} else {
|
||||
console.error('Error playing audio:', error);
|
||||
}
|
||||
});
|
||||
// .finally(() => {
|
||||
// resolve();
|
||||
// });
|
||||
audioDOM.onerror = () => {
|
||||
console.error('播放失败时间', +new Date());
|
||||
resolve();
|
||||
};
|
||||
audioDOM.onended = () => {
|
||||
console.error('播放结束时间: ', +new Date());
|
||||
// URL.revokeObjectURL(url);
|
||||
resolve();
|
||||
};
|
||||
});
|
||||
};
|
||||
// 当队列中任务数大于0时，开始处理队列中的任务
|
||||
const addQueue = (time, item) => {
|
||||
taskQueue.value.push({ func: item, time });
|
||||
if (taskQueue.value.length > 0 && !running.value) {
|
||||
running.value = true;
|
||||
processQueue();
|
||||
}
|
||||
};
|
||||
const processQueue = () => {
|
||||
const item = taskQueue.value.shift();
|
||||
if (item?.func) {
|
||||
item.func()
|
||||
.then(res => {
|
||||
console.log('已处理事件: ', res);
|
||||
})
|
||||
.finally(() => processQueue());
|
||||
} else {
|
||||
running.value = false;
|
||||
}
|
||||
};
|
||||
const destroyVideoStream = () => {
|
||||
videoStream.value?.getTracks().forEach(track => track.stop());
|
||||
videoStream.value = null;
|
||||
// 将srcObject设置为null以切断与MediaStream 对象的链接，以便将其释放
|
||||
videoRef.value.srcObject = null;
|
||||
|
||||
videoImage.value = [];
|
||||
videoLoaded.value = false;
|
||||
|
||||
clearInterval(intervalId);
|
||||
clearInterval(interval.value);
|
||||
interval.value = null;
|
||||
};
|
||||
const dealImage = () => {
|
||||
if (!videoRef.value) {
|
||||
return;
|
||||
}
|
||||
const canvas = canvasRef.value;
|
||||
canvasRef.value.width = videoRef.value.videoWidth;
|
||||
canvasRef.value.height = videoRef.value.videoHeight;
|
||||
const context = canvas.getContext('2d');
|
||||
context.drawImage(videoRef.value, 0, 0, canvasRef.value.width, canvasRef.value.height);
|
||||
const imageDataUrl = canvas.toDataURL('image/webp', 0.8);
|
||||
|
||||
videoImage.value.push({ src: imageDataUrl });
|
||||
};
|
||||
const drawBars = () => {
|
||||
// AnalyserNode接口的 getByteFrequencyData() 方法将当前频率数据复制到传入的 Uint8Array（无符号字节数组）中。
|
||||
analyser.value.getByteFrequencyData(dataArray.value);
|
||||
animationFrameId.value = requestAnimationFrame(drawBars);
|
||||
};
|
||||
// 跳过当前片段
|
||||
const skipVoice = async () => {
|
||||
// 打断前记录一下打断时间或vad触发事件
|
||||
vadStartTime.value = +new Date();
|
||||
if (!skipDisabled.value) {
|
||||
if (
|
||||
outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
|
||||
outputData.value[outputData.value.length - 1].audio === ''
|
||||
) {
|
||||
outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
|
||||
}
|
||||
base64List.value = [];
|
||||
audioPlayQueue.value = [];
|
||||
// 跳过之后，只保留当前时间点两秒内到之后的音频片段
|
||||
console.log(
|
||||
'截取前长度:',
|
||||
taskQueue.value.map(item => item.time)
|
||||
);
|
||||
let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
|
||||
if (startIndex !== -1) {
|
||||
taskQueue.value = taskQueue.value.slice(startIndex);
|
||||
console.log(
|
||||
'截取后长度:',
|
||||
taskQueue.value.map(item => item.time),
|
||||
vadStartTime.value
|
||||
);
|
||||
}
|
||||
stop.value = true;
|
||||
audioDOM?.pause();
|
||||
setTimeout(() => {
|
||||
skipDisabled.value = true;
|
||||
}, 300);
|
||||
try {
|
||||
playing.value = false;
|
||||
await stopMessage();
|
||||
stop.value = false;
|
||||
// playing.value = false;
|
||||
buildConnect();
|
||||
// cancelAnimationFrame(animationFrameId.value);
|
||||
} catch (err) {}
|
||||
}
|
||||
};
|
||||
// 每次call先上传当前用户配置
|
||||
const uploadUserConfig = async () => {
|
||||
if (!localStorage.getItem('configData')) {
|
||||
return new Promise(resolve => resolve());
|
||||
}
|
||||
const {
|
||||
videoQuality,
|
||||
useAudioPrompt,
|
||||
voiceClonePrompt,
|
||||
assistantPrompt,
|
||||
vadThreshold,
|
||||
audioFormat,
|
||||
base64Str
|
||||
} = JSON.parse(localStorage.getItem('configData'));
|
||||
const obj = {
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'input_audio',
|
||||
input_audio: {
|
||||
data: base64Str,
|
||||
format: audioFormat
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'options',
|
||||
options: {
|
||||
hd_video: videoQuality,
|
||||
use_audio_prompt: useAudioPrompt,
|
||||
vad_threshold: vadThreshold,
|
||||
voice_clone_prompt: voiceClonePrompt,
|
||||
assistant_prompt: assistantPrompt
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
};
|
||||
const { code, message, data } = await uploadConfig(obj);
|
||||
modelVersion.value = data?.choices?.content || '';
|
||||
return new Promise((resolve, reject) => {
|
||||
if (code !== 0) {
|
||||
ElMessage({
|
||||
type: 'error',
|
||||
message: message,
|
||||
duration: 3000,
|
||||
customClass: 'system-error'
|
||||
});
|
||||
reject();
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
};
|
||||
</script>
|
||||
<style lang="less" scoped>
|
||||
.video-page {
|
||||
flex: 1;
|
||||
height: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
&-header {
|
||||
width: 100%;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 0 16px 16px;
|
||||
box-shadow: 0 0.5px 0 0 #e0e0e0;
|
||||
margin-bottom: 16px;
|
||||
.header-icon {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
img {
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
margin-right: 8px;
|
||||
}
|
||||
span {
|
||||
color: rgba(23, 23, 23, 0.9);
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 500;
|
||||
line-height: normal;
|
||||
margin-right: 40px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
}
|
||||
.voice-container {
|
||||
display: flex;
|
||||
.voice-icon {
|
||||
width: 191px;
|
||||
height: 45px;
|
||||
}
|
||||
}
|
||||
}
|
||||
&-content {
|
||||
flex: 1;
|
||||
margin-bottom: 16px;
|
||||
display: flex;
|
||||
height: 0;
|
||||
&-video {
|
||||
width: 50%;
|
||||
height: 100%;
|
||||
background: #f3f3f3;
|
||||
flex-shrink: 0;
|
||||
position: relative;
|
||||
video {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: contain;
|
||||
}
|
||||
.switch-camera {
|
||||
position: absolute;
|
||||
top: 10px;
|
||||
right: 10px;
|
||||
width: 36px;
|
||||
height: 36px;
|
||||
background: #ffffff;
|
||||
border-radius: 6px;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
font-size: 24px;
|
||||
z-index: 999;
|
||||
.icon {
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
}
|
||||
}
|
||||
}
|
||||
&-right {
|
||||
margin-left: 16px;
|
||||
flex: 1;
|
||||
padding: 0 16px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
.output-content {
|
||||
flex: 1;
|
||||
overflow: auto;
|
||||
}
|
||||
.skip-box {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: flex-end;
|
||||
margin-top: 16px;
|
||||
}
|
||||
}
|
||||
}
|
||||
&-btn {
|
||||
text-align: center;
|
||||
padding: 8px 0;
|
||||
.el-button {
|
||||
width: 284px;
|
||||
height: 46px;
|
||||
border-radius: 8px;
|
||||
}
|
||||
.el-button.el-button--success {
|
||||
background: #647fff;
|
||||
border-color: #647fff;
|
||||
&:hover {
|
||||
opacity: 0.8;
|
||||
}
|
||||
span {
|
||||
color: #fff;
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 500;
|
||||
line-height: normal;
|
||||
}
|
||||
}
|
||||
.el-button.el-button--success.is-disabled {
|
||||
background: #f3f3f3;
|
||||
border-color: #f3f3f3;
|
||||
span {
|
||||
color: #d1d1d1;
|
||||
}
|
||||
}
|
||||
.el-button.el-button--danger {
|
||||
border-color: #dc3545;
|
||||
background-color: #dc3545;
|
||||
color: #ffffff;
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 500;
|
||||
line-height: normal;
|
||||
.phone-icon {
|
||||
margin-right: 10px;
|
||||
}
|
||||
.btn-text {
|
||||
margin-right: 10px;
|
||||
}
|
||||
.btn-desc {
|
||||
margin-right: 16px;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.video-size {
|
||||
position: absolute;
|
||||
bottom: 10px;
|
||||
right: 10px;
|
||||
background: rgba(0, 0, 0, 0.5);
|
||||
color: #fff;
|
||||
padding: 4px 8px;
|
||||
border-radius: 4px;
|
||||
font-size: 12px;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,955 @@
|
||||
<template>
|
||||
<ExtraInfo webVersion="websocket_0107" :modelVersion="modelVersion" />
|
||||
<div class="video-page">
|
||||
<div class="video-page-header">
|
||||
<div style="display: flex; align-items: center" class="header-icon">
|
||||
<img src="@/assets/images/voice-icon.png" />
|
||||
<span>Audio Choice</span>
|
||||
</div>
|
||||
<div class="voice-container" v-if="!isCalling">
|
||||
<SvgIcon name="voice" class="voice-icon" />
|
||||
<SvgIcon name="voice" class="voice-icon" />
|
||||
<SvgIcon name="voice" class="voice-icon" />
|
||||
</div>
|
||||
<div class="voice-container" v-else>
|
||||
<Voice
|
||||
:dataArray="dataArray"
|
||||
:isCalling="isCalling"
|
||||
:isPlaying="playing"
|
||||
:configList="videoConfigList"
|
||||
:boxStyle="{ height: '45px' }"
|
||||
:itemStyle="{ width: '3px', margin: '0 1px' }"
|
||||
/>
|
||||
</div>
|
||||
<!-- <SelectTimbre v-model:timbre="timbre" v-model:audioData="audioData" v-model:disabled="isCalling" /> -->
|
||||
</div>
|
||||
<div class="video-page-content">
|
||||
<div class="video-page-content-video" v-loading="loading" element-loading-background="#f3f3f3">
|
||||
<video ref="videoRef" autoplay playsinline muted />
|
||||
<canvas ref="canvasRef" canvas-id="canvasId" style="display: none" />
|
||||
<div class="switch-camera" v-if="isMobile()" @click="switchCamera">
|
||||
<SvgIcon name="switch-camera" class="icon" />
|
||||
</div>
|
||||
<!-- <div class="video-size" v-if="width || height">{{ width }} x {{ height }}</div> -->
|
||||
</div>
|
||||
<div class="video-page-content-right">
|
||||
<div class="output-content">
|
||||
<ModelOutput
|
||||
v-if="outputData.length > 0"
|
||||
:outputData="outputData"
|
||||
containerClass="output-content"
|
||||
/>
|
||||
</div>
|
||||
<div class="skip-box">
|
||||
<DelayTips
|
||||
v-if="delayTimestamp > 200 || delayCount > 2"
|
||||
:delayTimestamp="delayTimestamp"
|
||||
:delayCount="delayCount"
|
||||
/>
|
||||
<LikeAndDislike v-model:feedbackStatus="feedbackStatus" v-model:curResponseId="curResponseId" />
|
||||
<SkipBtn :disabled="skipDisabled" @click="skipVoice" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="video-page-btn">
|
||||
<el-button v-show="!isCalling" type="success" :disabled="callDisabled" @click="initRecording">
|
||||
{{ callDisabled ? 'Not ready yet, please wait' : 'Call MiniCPM' }}
|
||||
</el-button>
|
||||
<el-button v-show="isCalling" @click="stopRecording" type="danger">
|
||||
<SvgIcon name="phone-icon" className="phone-icon" />
|
||||
<span class="btn-text">Hang Up</span>
|
||||
<CountDown v-model="isCalling" @timeUp="stopRecording" />
|
||||
</el-button>
|
||||
</div>
|
||||
<IdeasList v-if="showIdeasList" :ideasList="videoIdeasList" />
|
||||
</div>
|
||||
</template>
|
||||
<script setup>
|
||||
import { sendMessage, stopMessage, uploadConfig } from '@/apis';
|
||||
import { encodeWAV } from '@/hooks/useVoice';
|
||||
import { getNewUserId, setNewUserId } from '@/hooks/useRandomId';
|
||||
import { fetchEventSource } from '@microsoft/fetch-event-source';
|
||||
import { MicVAD } from '@ricky0123/vad-web';
|
||||
import { videoIdeasList, videoConfigList, showIdeasList } from '@/enums';
|
||||
import { isMobile, maxCount, getChunkLength } from '@/utils';
|
||||
import { mergeBase64ToBlob } from './merge';
|
||||
import WebSocketService from '@/utils/websocket';
|
||||
let ctrl = new AbortController();
|
||||
let socket = null;
|
||||
const audioData = ref({
|
||||
base64Str: '',
|
||||
type: 'mp3'
|
||||
}); // 自定义音色base64
|
||||
const isCalling = defineModel();
|
||||
const videoRef = ref();
|
||||
const videoStream = ref(null);
|
||||
const interval = ref();
|
||||
const canvasRef = ref();
|
||||
const videoImage = ref([]);
|
||||
const videoLoaded = ref(false);
|
||||
const taskQueue = ref([]);
|
||||
const running = ref(false);
|
||||
const outputData = ref([]);
|
||||
const isFirstReturn = ref(true);
|
||||
const audioPlayQueue = ref([]);
|
||||
const base64List = ref([]);
|
||||
const playing = ref(false);
|
||||
const timbre = ref([1]);
|
||||
const isReturnError = ref(false);
|
||||
|
||||
const textQueue = ref('');
|
||||
const textAnimationInterval = ref();
|
||||
const analyser = ref();
|
||||
const dataArray = ref();
|
||||
const animationFrameId = ref();
|
||||
const skipDisabled = ref(true);
|
||||
const stop = ref(false);
|
||||
const isFrontCamera = ref(true);
|
||||
const loading = ref(false);
|
||||
const isEnd = ref(false); // sse接口关闭，认为模型已完成本次返回
|
||||
const isFirstPiece = ref(true);
|
||||
const allVoice = ref([]);
|
||||
const callDisabled = ref(true);
|
||||
const feedbackStatus = ref('');
|
||||
const curResponseId = ref('');
|
||||
const delayTimestamp = ref(0); // 当前发送片延时
|
||||
const delayCount = ref(0); // 当前剩余多少ms未发送到接口
|
||||
|
||||
const modelVersion = ref('');
|
||||
|
||||
let mediaStream;
|
||||
let audioRecorder;
|
||||
let audioStream;
|
||||
let audioContext;
|
||||
let audioChunks = [];
|
||||
let count = 0;
|
||||
let audioDOM;
|
||||
onBeforeUnmount(() => {
|
||||
stopRecording();
|
||||
});
|
||||
const vadStartTime = ref();
|
||||
let myvad = null;
|
||||
let vadTimer = null; // vad定时器，用于检测1s内人声是否停止，1s内停止，可认为是vad误触，直接忽略，1s内未停止，则认为是人声，已自动跳过当前对话
|
||||
const vadStart = async () => {
|
||||
myvad = await MicVAD.new({
|
||||
onSpeechStart: () => {
|
||||
console.log('Speech start', +new Date());
|
||||
if (!skipDisabled.value) {
|
||||
vadTimer && clearTimeout(vadTimer);
|
||||
vadTimer = setTimeout(() => {
|
||||
// vadStartTime.value = +new Date();
|
||||
console.log('打断时间: ', +new Date());
|
||||
skipVoice();
|
||||
}, 1000);
|
||||
}
|
||||
},
|
||||
onSpeechEnd: audio => {
|
||||
vadTimer && clearTimeout(vadTimer);
|
||||
console.log('Speech end', +new Date());
|
||||
// debugger;
|
||||
// do something with `audio` (Float32Array of audio samples at sample rate 16000)...
|
||||
}
|
||||
});
|
||||
myvad.start();
|
||||
};
|
||||
onMounted(async () => {
|
||||
const { code, message } = await stopMessage();
|
||||
if (code !== 0) {
|
||||
ElMessage({
|
||||
type: 'error',
|
||||
message: message,
|
||||
duration: 3000,
|
||||
customClass: 'system-error'
|
||||
});
|
||||
return;
|
||||
}
|
||||
callDisabled.value = false;
|
||||
});
|
||||
const delay = ms => {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
};
|
||||
const initRecording = async () => {
|
||||
uploadUserConfig()
|
||||
.then(async () => {
|
||||
if (!audioDOM) {
|
||||
audioDOM = new Audio();
|
||||
audioDOM.playsinline = true;
|
||||
audioDOM.preload = 'auto';
|
||||
}
|
||||
// 每次call都需要生成新uid
|
||||
setNewUserId();
|
||||
buildConnect();
|
||||
await delay(100);
|
||||
initVideoStream('environment');
|
||||
if (socket) {
|
||||
socket.close();
|
||||
}
|
||||
socket = new WebSocketService(
|
||||
`/ws/stream${window.location.search}&uid=${getNewUserId()}&service=minicpmo-server`
|
||||
);
|
||||
socket.connect();
|
||||
initVideoStream('environment');
|
||||
if (localStorage.getItem('canStopByVoice') === 'true') {
|
||||
vadStart();
|
||||
}
|
||||
})
|
||||
.catch(() => {});
|
||||
};
|
||||
// 切换摄像头
|
||||
const switchCamera = () => {
|
||||
if (!isCalling.value) {
|
||||
return;
|
||||
}
|
||||
isFrontCamera.value = !isFrontCamera.value;
|
||||
const facingMode = isFrontCamera.value ? 'environment' : 'user'; // 'user' 前置, 'environment' 后置
|
||||
initVideoStream(facingMode);
|
||||
};
|
||||
const initVideoStream = async facingMode => {
|
||||
if (mediaStream) {
|
||||
mediaStream.getTracks().forEach(track => track.stop());
|
||||
videoStream.value = null;
|
||||
}
|
||||
outputData.value = [];
|
||||
isCalling.value = true;
|
||||
loading.value = true;
|
||||
if (!videoStream.value) {
|
||||
try {
|
||||
mediaStream = await window.navigator.mediaDevices.getUserMedia({
|
||||
video: { facingMode },
|
||||
audio: true
|
||||
});
|
||||
console.log('mediaStream', mediaStream);
|
||||
videoStream.value = mediaStream;
|
||||
videoRef.value.srcObject = mediaStream;
|
||||
loading.value = false;
|
||||
console.log('打开后： ', +new Date());
|
||||
// takePhotos();
|
||||
audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 });
|
||||
console.log('samplate: ', audioContext);
|
||||
const audioSource = audioContext.createMediaStreamSource(mediaStream);
|
||||
interval.value = setInterval(() => dealImage(), 50);
|
||||
// 创建 ScriptProcessorNode 用于捕获音频数据
|
||||
const processor = audioContext.createScriptProcessor(256, 1, 1);
|
||||
processor.onaudioprocess = event => {
|
||||
if (!isCalling.value) return;
|
||||
if (isReturnError.value) {
|
||||
stopRecording();
|
||||
return;
|
||||
}
|
||||
const data = event.inputBuffer.getChannelData(0);
|
||||
audioChunks.push(new Float32Array(data));
|
||||
// 检查是否已经收集到1秒钟的数据
|
||||
const totalBufferLength = audioChunks.reduce((total, curr) => total + curr.length, 0);
|
||||
// const chunkLength = audioContext.sampleRate;
|
||||
const chunkLength = getChunkLength(audioContext.sampleRate);
|
||||
if (totalBufferLength >= chunkLength) {
|
||||
// 合并到一个完整的数据数组，并裁剪成1秒钟
|
||||
const mergedBuffer = mergeBuffers(audioChunks, totalBufferLength);
|
||||
const oneSecondBuffer = mergedBuffer.slice(0, audioContext.sampleRate);
|
||||
|
||||
// 保存并处理成WAV格式
|
||||
addQueue(+new Date(), () => saveAudioChunk(oneSecondBuffer, +new Date()));
|
||||
|
||||
// 保留多余的数据备用
|
||||
audioChunks = [mergedBuffer.slice(audioContext.sampleRate)];
|
||||
}
|
||||
};
|
||||
analyser.value = audioContext.createAnalyser();
|
||||
// 将音频节点连接到分析器
|
||||
audioSource.connect(analyser.value);
|
||||
// 分析器设置
|
||||
analyser.value.fftSize = 256;
|
||||
const bufferLength = analyser.value.frequencyBinCount;
|
||||
dataArray.value = new Uint8Array(bufferLength);
|
||||
// 开始绘制音波
|
||||
drawBars();
|
||||
|
||||
audioSource.connect(processor);
|
||||
processor.connect(audioContext.destination);
|
||||
} catch {}
|
||||
}
|
||||
};
|
||||
const drawText = async () => {
|
||||
if (textQueue.value.length > 0) {
|
||||
outputData.value[outputData.value.length - 1].text += textQueue.value[0];
|
||||
textQueue.value = textQueue.value.slice(1);
|
||||
} else {
|
||||
cancelAnimationFrame(textAnimationInterval.value);
|
||||
}
|
||||
textAnimationInterval.value = requestAnimationFrame(drawText);
|
||||
};
|
||||
const getStopValue = () => {
|
||||
return stop.value;
|
||||
};
|
||||
const getPlayingValue = () => {
|
||||
return playing.value;
|
||||
};
|
||||
const getStopStatus = () => {
|
||||
return localStorage.getItem('canStopByVoice') === 'true';
|
||||
};
|
||||
const saveAudioChunk = (buffer, timestamp) => {
|
||||
return new Promise(resolve => {
|
||||
if (!getStopStatus() && getPlayingValue()) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const wavBlob = encodeWAV(buffer, audioContext.sampleRate);
|
||||
let reader = new FileReader();
|
||||
reader.readAsDataURL(wavBlob);
|
||||
reader.onloadend = async function () {
|
||||
let base64data = reader.result.split(',')[1];
|
||||
const imgBase64 = videoImage.value[videoImage.value.length - 1]?.src;
|
||||
if (!(base64data && imgBase64)) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const strBase64 = imgBase64.split(',')[1];
|
||||
count++;
|
||||
let obj = {
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'input_audio',
|
||||
input_audio: {
|
||||
data: base64data,
|
||||
format: 'wav',
|
||||
timestamp: String(timestamp)
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
};
|
||||
obj.messages[0].content.unshift({
|
||||
type: 'image_data',
|
||||
image_data: {
|
||||
data: count === maxCount ? strBase64 : '',
|
||||
type: 2
|
||||
}
|
||||
});
|
||||
if (count === maxCount) {
|
||||
count = 0;
|
||||
}
|
||||
socket.send(JSON.stringify(obj));
|
||||
socket.on('message', data => {
|
||||
console.log('message: ', data);
|
||||
delayTimestamp.value = +new Date() - timestamp;
|
||||
delayCount.value = taskQueue.value.length;
|
||||
resolve();
|
||||
});
|
||||
// 将Base64音频数据发送到后端
|
||||
// try {
|
||||
// await sendMessage(obj);
|
||||
// delayTimestamp.value = +new Date() - timestamp;
|
||||
// delayCount.value = taskQueue.value.length;
|
||||
// } catch (err) {}
|
||||
// resolve();
|
||||
};
|
||||
});
|
||||
};
|
||||
const mergeBuffers = (buffers, length) => {
|
||||
const result = new Float32Array(length);
|
||||
let offset = 0;
|
||||
for (let buffer of buffers) {
|
||||
result.set(buffer, offset);
|
||||
offset += buffer.length;
|
||||
}
|
||||
return result;
|
||||
};
|
||||
const stopRecording = () => {
|
||||
isCalling.value = false;
|
||||
clearInterval(interval.value);
|
||||
interval.value = null;
|
||||
if (audioRecorder && audioRecorder.state !== 'inactive') {
|
||||
audioRecorder.stop();
|
||||
}
|
||||
if (animationFrameId.value) {
|
||||
cancelAnimationFrame(animationFrameId.value);
|
||||
}
|
||||
if (audioContext && audioContext.state !== 'closed') {
|
||||
audioContext.close();
|
||||
}
|
||||
destroyVideoStream();
|
||||
taskQueue.value = [];
|
||||
audioPlayQueue.value = [];
|
||||
base64List.value = [];
|
||||
ctrl.abort();
|
||||
ctrl = new AbortController();
|
||||
isReturnError.value = false;
|
||||
skipDisabled.value = true;
|
||||
playing.value = false;
|
||||
audioDOM?.pause();
|
||||
stopMessage();
|
||||
if (socket) {
|
||||
socket.close();
|
||||
}
|
||||
if (
|
||||
outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
|
||||
outputData.value[outputData.value.length - 1].audio === '' &&
|
||||
allVoice.value.length > 0
|
||||
) {
|
||||
outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
|
||||
}
|
||||
myvad && myvad.destroy();
|
||||
};
|
||||
// 建立连接
|
||||
const buildConnect = () => {
|
||||
const obj = {
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [{ type: 'none' }]
|
||||
}
|
||||
],
|
||||
stream: true
|
||||
};
|
||||
isEnd.value = false;
|
||||
ctrl.abort();
|
||||
ctrl = new AbortController();
|
||||
const url = `/api/v1/completions${window.location.search}`;
|
||||
|
||||
fetchEventSource(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
service: 'minicpmo-server',
|
||||
uid: getNewUserId()
|
||||
},
|
||||
body: JSON.stringify(obj),
|
||||
signal: ctrl.signal,
|
||||
openWhenHidden: true,
|
||||
async onopen(response) {
|
||||
isFirstPiece.value = true;
|
||||
isFirstReturn.value = true;
|
||||
allVoice.value = [];
|
||||
base64List.value = [];
|
||||
console.log('onopen', response);
|
||||
if (response.status !== 200) {
|
||||
ElMessage({
|
||||
type: 'error',
|
||||
message: 'At limit. Please try again soon.',
|
||||
duration: 3000,
|
||||
customClass: 'system-error'
|
||||
});
|
||||
isReturnError.value = true;
|
||||
} else {
|
||||
isReturnError.value = false;
|
||||
drawText();
|
||||
}
|
||||
},
|
||||
onmessage(msg) {
|
||||
const data = JSON.parse(msg.data);
|
||||
if (data.response_id) {
|
||||
curResponseId.value = data.response_id;
|
||||
}
|
||||
if (data.choices[0]?.text) {
|
||||
textQueue.value += data.choices[0].text.replace('<end>', '');
|
||||
console.warn('text return time -------------------------------', +new Date());
|
||||
}
|
||||
// 首次返回的是前端发给后端的音频片段，需要单独处理
|
||||
if (isFirstReturn.value) {
|
||||
console.log('第一次');
|
||||
isFirstReturn.value = false;
|
||||
// 如果后端返回的音频为空，需要重连
|
||||
if (!data.choices[0].audio) {
|
||||
buildConnect();
|
||||
return;
|
||||
}
|
||||
outputData.value.push({
|
||||
type: 'USER',
|
||||
audio: `data:audio/wav;base64,${data.choices[0].audio}`
|
||||
});
|
||||
outputData.value.push({
|
||||
type: 'BOT',
|
||||
text: '',
|
||||
audio: ''
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (data.choices[0]?.audio) {
|
||||
console.log('audio return time -------------------------------', +new Date());
|
||||
if (!getStopValue() && isCalling.value) {
|
||||
skipDisabled.value = false;
|
||||
base64List.value.push(`data:audio/wav;base64,${data.choices[0].audio}`);
|
||||
addAudioQueue(() => truePlay(data.choices[0].audio));
|
||||
}
|
||||
allVoice.value.push(`data:audio/wav;base64,${data.choices[0].audio}`);
|
||||
} else {
|
||||
// 发生异常了，直接重连
|
||||
buildConnect();
|
||||
}
|
||||
if (data.choices[0].text.includes('<end>')) {
|
||||
console.log('收到结束标记了:', +new Date());
|
||||
if (
|
||||
outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
|
||||
outputData.value[outputData.value.length - 1].audio === '' &&
|
||||
allVoice.value.length > 0
|
||||
) {
|
||||
outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
|
||||
}
|
||||
}
|
||||
},
|
||||
onclose() {
|
||||
console.log('onclose', +new Date());
|
||||
isEnd.value = true;
|
||||
outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
|
||||
// sse关闭后，如果待播放的音频列表为空，说明模型出错了，此次连接没有返回音频，则直接重连
|
||||
vadStartTime.value = +new Date();
|
||||
if (audioPlayQueue.value.length === 0) {
|
||||
let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
|
||||
console.log('taskQueue111111111: ', taskQueue.value, startIndex);
|
||||
if (startIndex !== -1) {
|
||||
taskQueue.value = taskQueue.value.slice(startIndex);
|
||||
console.log('截取后长度:', taskQueue.value, vadStartTime.value);
|
||||
}
|
||||
buildConnect();
|
||||
}
|
||||
},
|
||||
onerror(err) {
|
||||
console.log('onerror', err);
|
||||
ctrl.abort();
|
||||
ctrl = new AbortController();
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
};
|
||||
// 返回的语音放到队列里，挨个播放
|
||||
const addAudioQueue = async item => {
|
||||
audioPlayQueue.value.push(item);
|
||||
if (isFirstPiece.value) {
|
||||
await delay(1500);
|
||||
isFirstPiece.value = false;
|
||||
}
|
||||
if (audioPlayQueue.value.length > 0 && !playing.value) {
|
||||
playing.value = true;
|
||||
playAudio();
|
||||
}
|
||||
};
|
||||
// 控制播放队列执行
|
||||
const playAudio = () => {
|
||||
console.log('剩余播放列表:', audioPlayQueue.value, +new Date());
|
||||
|
||||
if (!isEnd.value && base64List.value.length >= 2) {
|
||||
const remainLen = base64List.value.length;
|
||||
const blob = mergeBase64ToBlob(base64List.value);
|
||||
audioDOM.src = blob;
|
||||
audioDOM.play();
|
||||
console.error('前期合并后播放开始时间: ', +new Date());
|
||||
audioDOM.onended = () => {
|
||||
console.error('前期合并后播放结束时间: ', +new Date());
|
||||
base64List.value = base64List.value.slice(remainLen);
|
||||
audioPlayQueue.value = audioPlayQueue.value.slice(remainLen);
|
||||
playAudio();
|
||||
};
|
||||
return;
|
||||
}
|
||||
if (isEnd.value && base64List.value.length >= 2) {
|
||||
const blob = mergeBase64ToBlob(base64List.value);
|
||||
audioDOM.src = blob;
|
||||
audioDOM.play();
|
||||
console.error('合并后播放开始时间: ', +new Date());
|
||||
audioDOM.onended = () => {
|
||||
console.error('合并后播放结束时间: ', +new Date());
|
||||
// URL.revokeObjectURL(url);
|
||||
base64List.value = [];
|
||||
audioPlayQueue.value = [];
|
||||
playing.value = false;
|
||||
skipDisabled.value = true;
|
||||
if (isCalling.value && !isReturnError.value) {
|
||||
// skipDisabled.value = true;
|
||||
taskQueue.value = [];
|
||||
// 打断前记录一下打断时间或vad触发事件
|
||||
// vadStartTime.value = +new Date();
|
||||
// // 每次完成后只保留当前时刻往前推1s的语音
|
||||
// console.log(
|
||||
// '截取前长度:',
|
||||
// taskQueue.value.map(item => item.time)
|
||||
// );
|
||||
// let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
|
||||
// if (startIndex !== -1) {
|
||||
// taskQueue.value = taskQueue.value.slice(startIndex);
|
||||
// console.log(
|
||||
// '截取后长度:',
|
||||
// taskQueue.value.map(item => item.time),
|
||||
// vadStartTime.value
|
||||
// );
|
||||
// }
|
||||
buildConnect();
|
||||
}
|
||||
};
|
||||
return;
|
||||
}
|
||||
base64List.value.shift();
|
||||
const _truePlay = audioPlayQueue.value.shift();
|
||||
if (_truePlay) {
|
||||
_truePlay().finally(() => {
|
||||
playAudio();
|
||||
});
|
||||
} else {
|
||||
playing.value = false;
|
||||
if (isEnd.value) {
|
||||
console.warn('play done................');
|
||||
skipDisabled.value = true;
|
||||
}
|
||||
// 播放完成后且正在通话且接口未返回错误时开始下一次连接
|
||||
if (isEnd.value && isCalling.value && !isReturnError.value) {
|
||||
// skipDisabled.value = true;
|
||||
taskQueue.value = [];
|
||||
// // 跳过之后，只保留当前时间点两秒内到之后的音频片段
|
||||
// vadStartTime.value = +new Date();
|
||||
// console.log(
|
||||
// '截取前长度:',
|
||||
// taskQueue.value.map(item => item.time)
|
||||
// );
|
||||
// let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
|
||||
// if (startIndex !== -1) {
|
||||
// taskQueue.value = taskQueue.value.slice(startIndex);
|
||||
// console.log(
|
||||
// '截取后长度:',
|
||||
// taskQueue.value.map(item => item.time),
|
||||
// vadStartTime.value
|
||||
// );
|
||||
// }
|
||||
buildConnect();
|
||||
}
|
||||
}
|
||||
};
|
||||
// 播放音频
|
||||
const truePlay = voice => {
|
||||
console.log('promise: ', +new Date());
|
||||
return new Promise(resolve => {
|
||||
audioDOM.src = 'data:audio/wav;base64,' + voice;
|
||||
console.error('播放开始时间:', +new Date());
|
||||
audioDOM
|
||||
.play()
|
||||
.then(() => {
|
||||
console.log('Audio played successfully');
|
||||
})
|
||||
.catch(error => {
|
||||
if (error.name === 'NotAllowedError' || error.name === 'SecurityError') {
|
||||
console.error('User interaction required or permission issue:', error);
|
||||
// ElMessage.warning('音频播放失败');
|
||||
console.error('播放失败时间');
|
||||
// alert('Please interact with the page (like clicking a button) to enable audio playback.');
|
||||
} else {
|
||||
console.error('Error playing audio:', error);
|
||||
}
|
||||
});
|
||||
// .finally(() => {
|
||||
// resolve();
|
||||
// });
|
||||
audioDOM.onerror = () => {
|
||||
console.error('播放失败时间', +new Date());
|
||||
resolve();
|
||||
};
|
||||
audioDOM.onended = () => {
|
||||
console.error('播放结束时间: ', +new Date());
|
||||
// URL.revokeObjectURL(url);
|
||||
resolve();
|
||||
};
|
||||
});
|
||||
};
|
||||
// 当队列中任务数大于0时，开始处理队列中的任务
|
||||
const addQueue = (time, item) => {
|
||||
taskQueue.value.push({ func: item, time });
|
||||
if (taskQueue.value.length > 0 && !running.value) {
|
||||
running.value = true;
|
||||
processQueue();
|
||||
}
|
||||
};
|
||||
const processQueue = () => {
|
||||
const item = taskQueue.value.shift();
|
||||
if (item?.func) {
|
||||
item.func()
|
||||
.then(res => {
|
||||
console.log('已处理事件: ', res);
|
||||
})
|
||||
.finally(() => processQueue());
|
||||
} else {
|
||||
running.value = false;
|
||||
}
|
||||
};
|
||||
const destroyVideoStream = () => {
|
||||
videoStream.value?.getTracks().forEach(track => track.stop());
|
||||
videoStream.value = null;
|
||||
// 将srcObject设置为null以切断与MediaStream 对象的链接，以便将其释放
|
||||
videoRef.value.srcObject = null;
|
||||
|
||||
videoImage.value = [];
|
||||
videoLoaded.value = false;
|
||||
|
||||
clearInterval(interval.value);
|
||||
interval.value = null;
|
||||
};
|
||||
const dealImage = () => {
|
||||
if (!videoRef.value) {
|
||||
return;
|
||||
}
|
||||
const canvas = canvasRef.value;
|
||||
canvasRef.value.width = videoRef.value.videoWidth;
|
||||
canvasRef.value.height = videoRef.value.videoHeight;
|
||||
const context = canvas.getContext('2d');
|
||||
context.drawImage(videoRef.value, 0, 0, canvasRef.value.width, canvasRef.value.height);
|
||||
const imageDataUrl = canvas.toDataURL('image/webp', 0.8);
|
||||
videoImage.value.push({ src: imageDataUrl });
|
||||
};
|
||||
const drawBars = () => {
|
||||
// AnalyserNode接口的 getByteFrequencyData() 方法将当前频率数据复制到传入的 Uint8Array（无符号字节数组）中。
|
||||
analyser.value.getByteFrequencyData(dataArray.value);
|
||||
animationFrameId.value = requestAnimationFrame(drawBars);
|
||||
};
|
||||
// 跳过当前片段
|
||||
const skipVoice = async () => {
|
||||
// 打断前记录一下打断时间或vad触发事件
|
||||
vadStartTime.value = +new Date();
|
||||
if (!skipDisabled.value) {
|
||||
if (
|
||||
outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
|
||||
outputData.value[outputData.value.length - 1].audio === ''
|
||||
) {
|
||||
outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
|
||||
}
|
||||
base64List.value = [];
|
||||
audioPlayQueue.value = [];
|
||||
// 跳过之后，只保留当前时间点两秒内到之后的音频片段
|
||||
console.log(
|
||||
'截取前长度:',
|
||||
taskQueue.value.map(item => item.time)
|
||||
);
|
||||
let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
|
||||
if (startIndex !== -1) {
|
||||
taskQueue.value = taskQueue.value.slice(startIndex);
|
||||
console.log(
|
||||
'截取后长度:',
|
||||
taskQueue.value.map(item => item.time),
|
||||
vadStartTime.value
|
||||
);
|
||||
}
|
||||
stop.value = true;
|
||||
audioDOM?.pause();
|
||||
setTimeout(() => {
|
||||
skipDisabled.value = true;
|
||||
}, 300);
|
||||
try {
|
||||
playing.value = false;
|
||||
await stopMessage();
|
||||
stop.value = false;
|
||||
// playing.value = false;
|
||||
buildConnect();
|
||||
// cancelAnimationFrame(animationFrameId.value);
|
||||
} catch (err) {}
|
||||
}
|
||||
};
|
||||
// 每次call先上传当前用户配置
|
||||
const uploadUserConfig = async () => {
|
||||
if (!localStorage.getItem('configData')) {
|
||||
return new Promise(resolve => resolve());
|
||||
}
|
||||
const {
|
||||
videoQuality,
|
||||
useAudioPrompt,
|
||||
voiceClonePrompt,
|
||||
assistantPrompt,
|
||||
vadThreshold,
|
||||
audioFormat,
|
||||
base64Str
|
||||
} = JSON.parse(localStorage.getItem('configData'));
|
||||
const obj = {
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'input_audio',
|
||||
input_audio: {
|
||||
data: base64Str,
|
||||
format: audioFormat
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'options',
|
||||
options: {
|
||||
hd_video: videoQuality,
|
||||
use_audio_prompt: useAudioPrompt,
|
||||
vad_threshold: vadThreshold,
|
||||
voice_clone_prompt: voiceClonePrompt,
|
||||
assistant_prompt: assistantPrompt
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
};
|
||||
const { code, message, data } = await uploadConfig(obj);
|
||||
modelVersion.value = data?.choices?.content || '';
|
||||
return new Promise((resolve, reject) => {
|
||||
if (code !== 0) {
|
||||
ElMessage({
|
||||
type: 'error',
|
||||
message: message,
|
||||
duration: 3000,
|
||||
customClass: 'system-error'
|
||||
});
|
||||
reject();
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
};
|
||||
</script>
|
||||
<style lang="less">
|
||||
.video-page {
|
||||
height: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
&-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
padding: 0 16px 16px;
|
||||
box-shadow: 0 0.5px 0 0 #e0e0e0;
|
||||
margin-bottom: 16px;
|
||||
justify-content: space-between;
|
||||
.header-icon {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
img {
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
margin-right: 8px;
|
||||
}
|
||||
span {
|
||||
color: rgba(23, 23, 23, 0.9);
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 500;
|
||||
line-height: normal;
|
||||
margin-right: 40px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
}
|
||||
.voice-container {
|
||||
display: flex;
|
||||
.voice-icon {
|
||||
width: 191px;
|
||||
height: 45px;
|
||||
}
|
||||
}
|
||||
}
|
||||
&-content {
|
||||
flex: 1;
|
||||
margin-bottom: 16px;
|
||||
display: flex;
|
||||
height: 0;
|
||||
&-video {
|
||||
width: 50%;
|
||||
height: 100%;
|
||||
background: #f3f3f3;
|
||||
flex-shrink: 0;
|
||||
position: relative;
|
||||
video {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: contain;
|
||||
}
|
||||
.switch-camera {
|
||||
position: absolute;
|
||||
top: 10px;
|
||||
right: 10px;
|
||||
width: 36px;
|
||||
height: 36px;
|
||||
background: #ffffff;
|
||||
border-radius: 6px;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
font-size: 24px;
|
||||
z-index: 999;
|
||||
.icon {
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
}
|
||||
}
|
||||
}
|
||||
&-right {
|
||||
margin-left: 16px;
|
||||
flex: 1;
|
||||
padding: 0 16px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
.output-content {
|
||||
flex: 1;
|
||||
overflow: auto;
|
||||
}
|
||||
.skip-box {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: flex-end;
|
||||
margin-top: 16px;
|
||||
}
|
||||
}
|
||||
}
|
||||
&-btn {
|
||||
text-align: center;
|
||||
padding: 8px 0;
|
||||
.el-button {
|
||||
width: 284px;
|
||||
height: 46px;
|
||||
border-radius: 8px;
|
||||
}
|
||||
.el-button.el-button--success {
|
||||
background: #647fff;
|
||||
border-color: #647fff;
|
||||
&:hover {
|
||||
opacity: 0.8;
|
||||
}
|
||||
span {
|
||||
color: #fff;
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 500;
|
||||
line-height: normal;
|
||||
}
|
||||
}
|
||||
.el-button.el-button--success.is-disabled {
|
||||
background: #f3f3f3;
|
||||
border-color: #f3f3f3;
|
||||
span {
|
||||
color: #d1d1d1;
|
||||
}
|
||||
}
|
||||
.el-button.el-button--danger {
|
||||
border-color: #dc3545;
|
||||
background-color: #dc3545;
|
||||
color: #ffffff;
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 500;
|
||||
line-height: normal;
|
||||
.phone-icon {
|
||||
margin-right: 10px;
|
||||
}
|
||||
.btn-text {
|
||||
margin-right: 10px;
|
||||
}
|
||||
.btn-desc {
|
||||
margin-right: 16px;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.video-size {
|
||||
position: absolute;
|
||||
bottom: 10px;
|
||||
right: 10px;
|
||||
background: rgba(0, 0, 0, 0.5);
|
||||
color: #fff;
|
||||
padding: 4px 8px;
|
||||
border-radius: 4px;
|
||||
font-size: 12px;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,833 @@
|
||||
<template>
|
||||
<!-- <ExtraInfo webVersion="非websocket_0112" :modelVersion="modelVersion" /> -->
|
||||
<div class="voice-page">
|
||||
<div class="voice-page-header">
|
||||
<div class="voice-container" v-if="!isCalling">
|
||||
<SvgIcon name="voice" class="voice-icon" />
|
||||
<SvgIcon name="voice" class="voice-icon" />
|
||||
<SvgIcon name="voice" class="voice-icon" />
|
||||
</div>
|
||||
<div class="voice-container" v-else>
|
||||
<Voice
|
||||
:dataArray="dataArray"
|
||||
:isCalling="isCalling"
|
||||
:isPlaying="playing"
|
||||
:configList="videoConfigList"
|
||||
:boxStyle="{ height: '45px' }"
|
||||
:itemStyle="{ width: '3px', margin: '0 1px' }"
|
||||
/>
|
||||
</div>
|
||||
<!-- <SelectTimbre v-model:timbre="timbre" v-model:audioData="audioData" v-model:disabled="isCalling" /> -->
|
||||
</div>
|
||||
<div class="voice-page-output">
|
||||
<div class="output-content">
|
||||
<ModelOutput v-if="outputData.length > 0" :outputData="outputData" containerClass="output-content" />
|
||||
</div>
|
||||
<div class="skip-box">
|
||||
<!-- <DelayTips
|
||||
v-if="delayTimestamp > 200 || delayCount > 2"
|
||||
:delayTimestamp="delayTimestamp"
|
||||
:delayCount="delayCount"
|
||||
/> -->
|
||||
<LikeAndDislike v-model:feedbackStatus="feedbackStatus" v-model:curResponseId="curResponseId" />
|
||||
<SkipBtn :disabled="skipDisabled" @click="skipVoice" />
|
||||
</div>
|
||||
</div>
|
||||
<div class="voice-page-btn">
|
||||
<el-button v-show="!isCalling" type="success" :disabled="callDisabled" @click="initRecording">
|
||||
{{ callDisabled ? t('notReadyBtn') : t('audioCallBtn') }}
|
||||
</el-button>
|
||||
<el-button v-show="isCalling" @click="stopRecording" type="danger">
|
||||
<SvgIcon name="phone-icon" className="phone-icon" />
|
||||
<span class="btn-text">{{ t('hangUpBtn') }}</span>
|
||||
<CountDown v-model="isCalling" @timeUp="stopRecording" />
|
||||
</el-button>
|
||||
</div>
|
||||
<IdeasList v-if="showIdeasList" :ideasList="voiceIdeasList" />
|
||||
</div>
|
||||
</template>
|
||||
<script setup>
|
||||
import { sendMessage, stopMessage, uploadConfig } from '@/apis';
|
||||
import { encodeWAV } from '@/hooks/useVoice';
|
||||
import { getNewUserId, setNewUserId } from '@/hooks/useRandomId';
|
||||
import { fetchEventSource } from '@microsoft/fetch-event-source';
|
||||
import { MicVAD } from '@ricky0123/vad-web';
|
||||
import { videoConfigList, voiceConfigList, voiceIdeasList, showIdeasList } from '@/enums';
|
||||
import { getChunkLength } from '@/utils';
|
||||
import { mergeBase64ToBlob } from './merge';
|
||||
import WebSocketService from '@/utils/websocket';
|
||||
import { useI18n } from 'vue-i18n';
|
||||
|
||||
const { t } = useI18n();
|
||||
|
||||
let ctrl = new AbortController();
|
||||
let socket = null;
|
||||
const audioData = ref({
|
||||
base64Str: '',
|
||||
type: 'mp3'
|
||||
}); // 自定义音色base64
|
||||
const isCalling = defineModel();
|
||||
const taskQueue = ref([]);
|
||||
const running = ref(false);
|
||||
const outputData = ref([]);
|
||||
const textQueue = ref('');
|
||||
const textAnimationInterval = ref();
|
||||
|
||||
const isFirstReturn = ref(true); // 首次返回的音频是前端发给后端的音频片段，需要单独处理
|
||||
|
||||
const audioPlayQueue = ref([]);
|
||||
const base64List = ref([]);
|
||||
const playing = ref(false);
|
||||
const skipDisabled = ref(true);
|
||||
const stop = ref(false);
|
||||
const timbre = ref([1]);
|
||||
const isReturnError = ref(false);
|
||||
const allVoice = ref([]);
|
||||
const callDisabled = ref(true);
|
||||
|
||||
const feedbackStatus = ref('');
|
||||
const curResponseId = ref('');
|
||||
const delayTimestamp = ref(0); // 当前发送片延时
|
||||
const delayCount = ref(0); // 当前剩余多少ms未发送到接口
|
||||
|
||||
const modelVersion = ref('');
|
||||
|
||||
let audioDOM = new Audio();
|
||||
|
||||
const isEnd = ref(false); // sse接口关闭，认为模型已完成本次返回
|
||||
// 页面卸载时关闭录音
|
||||
onBeforeUnmount(() => {
|
||||
stopRecording();
|
||||
});
|
||||
const vadStartTime = ref();
|
||||
let myvad = null;
|
||||
let vadTimer = null; // vad定时器，用于检测1s内人声是否停止，1s内停止，可认为是vad误触，直接忽略，1s内未停止，则认为是人声，已自动跳过当前对话
|
||||
const vadStart = async () => {
|
||||
myvad = await MicVAD.new({
|
||||
onSpeechStart: () => {
|
||||
console.log('Speech start detected');
|
||||
// if (!skipDisabled.value) {
|
||||
vadTimer && clearTimeout(vadTimer);
|
||||
vadTimer = setTimeout(() => {
|
||||
console.log('打断时间: ', +new Date());
|
||||
skipVoice();
|
||||
}, 500);
|
||||
// }
|
||||
},
|
||||
onSpeechEnd: audio => {
|
||||
vadTimer && clearTimeout(vadTimer);
|
||||
// debugger;
|
||||
// do something with `audio` (Float32Array of audio samples at sample rate 16000)...
|
||||
},
|
||||
baseAssetPath: '/'
|
||||
});
|
||||
console.log('vad: ', myvad);
|
||||
myvad.start();
|
||||
};
|
||||
onMounted(async () => {
|
||||
const { code, message } = await stopMessage();
|
||||
if (code !== 0) {
|
||||
ElMessage({
|
||||
type: 'error',
|
||||
message: message,
|
||||
duration: 3000,
|
||||
customClass: 'system-error'
|
||||
});
|
||||
return;
|
||||
}
|
||||
callDisabled.value = false;
|
||||
});
|
||||
const delay = ms => {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
};
|
||||
const initRecording = async () => {
|
||||
uploadUserConfig()
|
||||
.then(async () => {
|
||||
// 每次call都需要生成新uid
|
||||
setNewUserId();
|
||||
|
||||
outputData.value = [];
|
||||
buildConnect();
|
||||
isCalling.value = true;
|
||||
await delay(100);
|
||||
// if (socket) {
|
||||
// socket.close();
|
||||
// }
|
||||
// socket = new WebSocketService(
|
||||
// `/ws/stream${window.location.search}&uid=${getNewUserId()}&service=minicpmo-server`
|
||||
// );
|
||||
// socket.connect();
|
||||
// 建立连接后稍等一会儿再传送数据
|
||||
startRecording();
|
||||
if (localStorage.getItem('canStopByVoice') === 'true') {
|
||||
vadStart();
|
||||
}
|
||||
})
|
||||
.catch(() => {});
|
||||
};
|
||||
let audioContext;
|
||||
const analyser = ref();
|
||||
const dataArray = ref();
|
||||
let mediaRecorder;
|
||||
let audioChunks = [];
|
||||
const animationFrameId = ref();
|
||||
|
||||
const isFirstPiece = ref(true);
|
||||
|
||||
const startRecording = async () => {
|
||||
// 获取用户音频流
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
|
||||
// 创建 AudioContext 和 MediaStreamAudioSourceNode
|
||||
audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 });
|
||||
const source = audioContext.createMediaStreamSource(stream);
|
||||
|
||||
analyser.value = audioContext.createAnalyser();
|
||||
// 将音频节点连接到分析器
|
||||
source.connect(analyser.value);
|
||||
// 分析器设置
|
||||
analyser.value.fftSize = 256;
|
||||
const bufferLength = analyser.value.frequencyBinCount;
|
||||
dataArray.value = new Uint8Array(bufferLength);
|
||||
// 开始绘制音波
|
||||
drawBars();
|
||||
|
||||
// 创建 ScriptProcessorNode 用于捕获音频数据
|
||||
const processor = audioContext.createScriptProcessor(256, 1, 1);
|
||||
|
||||
processor.onaudioprocess = event => {
|
||||
if (!isCalling.value) return;
|
||||
if (isReturnError.value) {
|
||||
stopRecording();
|
||||
return;
|
||||
}
|
||||
const data = event.inputBuffer.getChannelData(0);
|
||||
audioChunks.push(new Float32Array(data));
|
||||
// 检查是否已经收集到1秒钟的数据
|
||||
const totalBufferLength = audioChunks.reduce((total, curr) => total + curr.length, 0);
|
||||
const chunkLength = getChunkLength(audioContext.sampleRate);
|
||||
if (totalBufferLength >= chunkLength) {
|
||||
// 合并到一个完整的数据数组，并裁剪成1秒钟
|
||||
const mergedBuffer = mergeBuffers(audioChunks, totalBufferLength);
|
||||
const oneSecondBuffer = mergedBuffer.slice(0, chunkLength);
|
||||
// 保存并处理成WAV格式
|
||||
addQueue(+new Date(), () => saveAudioChunk(oneSecondBuffer, +new Date()));
|
||||
// 保留多余的数据备用
|
||||
audioChunks = [mergedBuffer.slice(chunkLength)];
|
||||
}
|
||||
};
|
||||
|
||||
source.connect(processor);
|
||||
processor.connect(audioContext.destination);
|
||||
};
|
||||
const stopRecording = () => {
|
||||
isCalling.value = false;
|
||||
if (animationFrameId.value) {
|
||||
cancelAnimationFrame(animationFrameId.value);
|
||||
}
|
||||
if (audioContext && audioContext.state !== 'closed') {
|
||||
audioContext.close();
|
||||
}
|
||||
ctrl.abort();
|
||||
ctrl = new AbortController();
|
||||
taskQueue.value = [];
|
||||
audioPlayQueue.value = [];
|
||||
base64List.value = [];
|
||||
isReturnError.value = false;
|
||||
skipDisabled.value = true;
|
||||
playing.value = false;
|
||||
audioDOM.pause();
|
||||
stopMessage();
|
||||
if (socket) {
|
||||
socket.close();
|
||||
}
|
||||
if (
|
||||
outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
|
||||
outputData.value[outputData.value.length - 1].audio === '' &&
|
||||
allVoice.value.length > 0
|
||||
) {
|
||||
outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
|
||||
}
|
||||
myvad && myvad.destroy();
|
||||
};
|
||||
const getStopValue = () => {
|
||||
return stop.value;
|
||||
};
|
||||
const getPlayingValue = () => {
|
||||
return playing.value;
|
||||
};
|
||||
const getStopStatus = () => {
|
||||
return localStorage.getItem('canStopByVoice') === 'true';
|
||||
};
|
||||
const saveAudioChunk = (buffer, timestamp) => {
|
||||
return new Promise(resolve => {
|
||||
if (!getStopStatus() && getPlayingValue()) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const wavBlob = encodeWAV(buffer, audioContext.sampleRate);
|
||||
let reader = new FileReader();
|
||||
reader.readAsDataURL(wavBlob);
|
||||
|
||||
reader.onloadend = async function () {
|
||||
let base64data = reader.result.split(',')[1];
|
||||
if (!base64data) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const obj = {
|
||||
uid: getNewUserId(),
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'input_audio',
|
||||
input_audio: {
|
||||
data: base64data,
|
||||
format: 'wav',
|
||||
timestamp: String(timestamp)
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
};
|
||||
// socket.send(JSON.stringify(obj));
|
||||
// socket.on('message', data => {
|
||||
// console.log('message: ', data);
|
||||
// delayTimestamp.value = +new Date() - timestamp;
|
||||
// delayCount.value = taskQueue.value.length;
|
||||
// resolve();
|
||||
// });
|
||||
// 将Base64音频数据发送到后端
|
||||
try {
|
||||
await sendMessage(obj);
|
||||
delayTimestamp.value = +new Date() - timestamp;
|
||||
delayCount.value = taskQueue.value.length;
|
||||
} catch (err) {}
|
||||
resolve();
|
||||
};
|
||||
});
|
||||
};
|
||||
const mergeBuffers = (buffers, length) => {
|
||||
const result = new Float32Array(length);
|
||||
let offset = 0;
|
||||
for (let buffer of buffers) {
|
||||
result.set(buffer, offset);
|
||||
offset += buffer.length;
|
||||
}
|
||||
return result;
|
||||
};
|
||||
// 建立连接
|
||||
const buildConnect = async () => {
|
||||
const obj = {
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [{ type: 'none' }]
|
||||
}
|
||||
],
|
||||
stream: true
|
||||
};
|
||||
isEnd.value = false;
|
||||
ctrl.abort();
|
||||
ctrl = new AbortController();
|
||||
const url = `/api/v1/completions${window.location.search}`;
|
||||
fetchEventSource(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
service: 'minicpmo-server',
|
||||
uid: getNewUserId()
|
||||
},
|
||||
body: JSON.stringify(obj),
|
||||
signal: ctrl.signal,
|
||||
openWhenHidden: true,
|
||||
async onopen(response) {
|
||||
console.log('onopen', response);
|
||||
isFirstPiece.value = true;
|
||||
isFirstReturn.value = true;
|
||||
allVoice.value = [];
|
||||
base64List.value = [];
|
||||
if (response.status !== 200) {
|
||||
ElMessage({
|
||||
type: 'error',
|
||||
message: 'At limit. Please try again soon.',
|
||||
duration: 3000,
|
||||
customClass: 'system-error'
|
||||
});
|
||||
isReturnError.value = true;
|
||||
} else {
|
||||
isReturnError.value = false;
|
||||
// skipDisabled.value = false;
|
||||
drawText();
|
||||
}
|
||||
},
|
||||
onmessage(msg) {
|
||||
const data = JSON.parse(msg.data);
|
||||
if (data.response_id) {
|
||||
curResponseId.value = data.response_id;
|
||||
}
|
||||
if (data.choices[0]?.text) {
|
||||
textQueue.value += data.choices[0].text.replace('<end>', '');
|
||||
console.warn('text return time -------------------------------', +new Date());
|
||||
}
|
||||
// 首次返回的是前端发给后端的音频片段，需要单独处理
|
||||
if (isFirstReturn.value) {
|
||||
console.log('第一次');
|
||||
isFirstReturn.value = false;
|
||||
// 如果后端返回的音频为空，需要重连
|
||||
if (!data.choices[0].audio) {
|
||||
buildConnect();
|
||||
return;
|
||||
}
|
||||
outputData.value.push({
|
||||
type: 'USER',
|
||||
audio: `data:audio/wav;base64,${data.choices[0].audio}`
|
||||
});
|
||||
outputData.value.push({
|
||||
type: 'BOT',
|
||||
text: '',
|
||||
audio: ''
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (data.choices[0]?.audio) {
|
||||
console.warn('audio return time -------------------------------', +new Date());
|
||||
if (!getStopValue() && isCalling.value) {
|
||||
skipDisabled.value = false;
|
||||
base64List.value.push(`data:audio/wav;base64,${data.choices[0].audio}`);
|
||||
addAudioQueue(() => truePlay(data.choices[0].audio));
|
||||
}
|
||||
allVoice.value.push(`data:audio/wav;base64,${data.choices[0].audio}`);
|
||||
} else {
|
||||
// 发生异常了，直接重连
|
||||
buildConnect();
|
||||
}
|
||||
if (data.choices[0].text.includes('<end>')) {
|
||||
// isEnd.value = true;
|
||||
console.log('收到结束标记了:', +new Date());
|
||||
if (
|
||||
outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
|
||||
outputData.value[outputData.value.length - 1].audio === '' &&
|
||||
allVoice.value.length > 0
|
||||
) {
|
||||
outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
|
||||
}
|
||||
}
|
||||
},
|
||||
onclose() {
|
||||
console.log('onclose', +new Date());
|
||||
isEnd.value = true;
|
||||
if (
|
||||
outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
|
||||
outputData.value[outputData.value.length - 1].audio === '' &&
|
||||
allVoice.value.length > 0
|
||||
) {
|
||||
outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
|
||||
}
|
||||
vadStartTime.value = +new Date();
|
||||
if (audioPlayQueue.value.length === 0) {
|
||||
let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 2000);
|
||||
console.log('taskQueue111111111: ', taskQueue.value, startIndex);
|
||||
if (startIndex !== -1) {
|
||||
taskQueue.value = taskQueue.value.slice(startIndex);
|
||||
console.log('截取后长度:', taskQueue.value, vadStartTime.value);
|
||||
}
|
||||
buildConnect();
|
||||
}
|
||||
},
|
||||
onerror(err) {
|
||||
console.log('onerror', err);
|
||||
ctrl.abort();
|
||||
ctrl = new AbortController();
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
};
|
||||
const drawText = async () => {
|
||||
if (textQueue.value.length > 0) {
|
||||
outputData.value[outputData.value.length - 1].text += textQueue.value[0];
|
||||
textQueue.value = textQueue.value.slice(1);
|
||||
} else {
|
||||
cancelAnimationFrame(textAnimationInterval.value);
|
||||
}
|
||||
textAnimationInterval.value = requestAnimationFrame(drawText);
|
||||
};
|
||||
// 返回的语音放到队列里，挨个播放
|
||||
const addAudioQueue = async item => {
|
||||
audioPlayQueue.value.push(item);
|
||||
if (isFirstPiece.value) {
|
||||
await delay(500);
|
||||
isFirstPiece.value = false;
|
||||
}
|
||||
if (audioPlayQueue.value.length > 0 && !playing.value) {
|
||||
playing.value = true;
|
||||
playAudio();
|
||||
}
|
||||
};
|
||||
// 控制播放队列执行
|
||||
const playAudio = () => {
|
||||
console.log('剩余播放列表:', audioPlayQueue.value, +new Date());
|
||||
|
||||
if (!isEnd.value && base64List.value.length >= 2) {
|
||||
const remainLen = base64List.value.length;
|
||||
const blob = mergeBase64ToBlob(base64List.value);
|
||||
audioDOM.src = blob;
|
||||
audioDOM.play();
|
||||
console.error('前期合并后播放开始时间: ', +new Date());
|
||||
audioDOM.onended = () => {
|
||||
console.error('前期合并后播放结束时间: ', +new Date());
|
||||
base64List.value = base64List.value.slice(remainLen);
|
||||
audioPlayQueue.value = audioPlayQueue.value.slice(remainLen);
|
||||
playAudio();
|
||||
};
|
||||
return;
|
||||
}
|
||||
if (isEnd.value && base64List.value.length >= 2) {
|
||||
const blob = mergeBase64ToBlob(base64List.value);
|
||||
// let audio = new Audio();
|
||||
audioDOM.src = blob;
|
||||
audioDOM.play();
|
||||
console.error('最后合并后播放开始时间: ', +new Date());
|
||||
audioDOM.onended = () => {
|
||||
console.error('合并后播放结束时间: ', +new Date());
|
||||
// URL.revokeObjectURL(url);
|
||||
base64List.value = [];
|
||||
audioPlayQueue.value = [];
|
||||
playing.value = false;
|
||||
skipDisabled.value = true;
|
||||
if (isCalling.value && !isReturnError.value) {
|
||||
// skipDisabled.value = true;
|
||||
taskQueue.value = [];
|
||||
// 打断前记录一下打断时间或vad触发事件
|
||||
// vadStartTime.value = +new Date();
|
||||
// // 每次完成后只保留当前时刻往前推1s的语音
|
||||
// console.log('截取前长度:', JSON.parse(JSON.stringify(taskQueue.value.map(item => item.time))));
|
||||
// let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 2000);
|
||||
// if (startIndex !== -1) {
|
||||
// taskQueue.value = taskQueue.value.slice(startIndex);
|
||||
// console.log(
|
||||
// '截取后长度:',
|
||||
// taskQueue.value.map(item => item.time),
|
||||
// vadStartTime.value
|
||||
// );
|
||||
// }
|
||||
buildConnect();
|
||||
}
|
||||
};
|
||||
return;
|
||||
}
|
||||
base64List.value.shift();
|
||||
const item = audioPlayQueue.value.shift();
|
||||
if (item) {
|
||||
item().finally(() => playAudio());
|
||||
} else {
|
||||
playing.value = false;
|
||||
if (isEnd.value) {
|
||||
console.warn('play done................');
|
||||
skipDisabled.value = true;
|
||||
}
|
||||
// 播放完成后且正在通话且接口未返回错误时开始下一次连接
|
||||
if (isEnd.value && isCalling.value && !isReturnError.value) {
|
||||
// skipDisabled.value = true;
|
||||
taskQueue.value = [];
|
||||
// 打断前记录一下打断时间或vad触发事件
|
||||
// vadStartTime.value = +new Date();
|
||||
// // 每次完成后只保留当前时刻往前推1s的语音
|
||||
// console.log(
|
||||
// '截取前长度:',
|
||||
// taskQueue.value.map(item => item.time)
|
||||
// );
|
||||
// let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 2000);
|
||||
// if (startIndex !== -1) {
|
||||
// taskQueue.value = taskQueue.value.slice(startIndex);
|
||||
// console.log(
|
||||
// '截取后长度:',
|
||||
// taskQueue.value.map(item => item.time),
|
||||
// vadStartTime.value
|
||||
// );
|
||||
// }
|
||||
buildConnect();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// 播放音频
|
||||
const truePlay = async voice => {
|
||||
return new Promise(resolve => {
|
||||
audioDOM.src = 'data:audio/wav;base64,' + voice;
|
||||
console.error('播放开始时间:', +new Date());
|
||||
audioDOM
|
||||
.play()
|
||||
.then(() => {
|
||||
// console.error('播放结束时间: ', +new Date());
|
||||
})
|
||||
.catch(error => {
|
||||
resolve();
|
||||
if (error.name === 'NotAllowedError' || error.name === 'SecurityError') {
|
||||
console.error('User interaction required or permission issue:', error);
|
||||
ElMessage.warning('音频播放失败');
|
||||
} else {
|
||||
console.error('Error playing audio:', error);
|
||||
}
|
||||
});
|
||||
|
||||
audioDOM.onended = () => {
|
||||
console.error('播放结束时间: ', +new Date());
|
||||
// URL.revokeObjectURL(url);
|
||||
resolve();
|
||||
};
|
||||
});
|
||||
};
|
||||
// 当队列中任务数大于0时，开始处理队列中的任务
|
||||
const addQueue = (time, item) => {
|
||||
taskQueue.value.push({ func: item, time });
|
||||
if (taskQueue.value.length > 0 && !running.value) {
|
||||
running.value = true;
|
||||
processQueue();
|
||||
}
|
||||
};
|
||||
const processQueue = () => {
|
||||
const item = taskQueue.value.shift();
|
||||
if (item?.func) {
|
||||
item.func().then(() => {
|
||||
console.warn('shift!!!!!!!!!');
|
||||
processQueue();
|
||||
});
|
||||
} else {
|
||||
running.value = false;
|
||||
}
|
||||
};
|
||||
const drawBars = () => {
|
||||
// AnalyserNode接口的 getByteFrequencyData() 方法将当前频率数据复制到传入的 Uint8Array（无符号字节数组）中。
|
||||
analyser.value.getByteFrequencyData(dataArray.value);
|
||||
animationFrameId.value = requestAnimationFrame(drawBars);
|
||||
};
|
||||
// 跳过当前片段
|
||||
const skipVoice = async () => {
|
||||
// 打断前记录一下打断时间或vad触发事件
|
||||
vadStartTime.value = +new Date();
|
||||
if (!skipDisabled.value) {
|
||||
if (
|
||||
outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
|
||||
outputData.value[outputData.value.length - 1].audio === ''
|
||||
) {
|
||||
outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
|
||||
}
|
||||
base64List.value = [];
|
||||
audioPlayQueue.value = [];
|
||||
// 跳过之后，只保留当前时间点两秒内到之后的音频片段
|
||||
console.log(
|
||||
'截取前长度:',
|
||||
taskQueue.value.map(item => item.time)
|
||||
);
|
||||
let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
|
||||
if (startIndex !== -1) {
|
||||
taskQueue.value = taskQueue.value.slice(startIndex);
|
||||
console.log(
|
||||
'截取后长度:',
|
||||
taskQueue.value.map(item => item.time),
|
||||
vadStartTime.value
|
||||
);
|
||||
}
|
||||
stop.value = true;
|
||||
audioDOM.pause();
|
||||
setTimeout(() => {
|
||||
skipDisabled.value = true;
|
||||
}, 300);
|
||||
try {
|
||||
playing.value = false;
|
||||
await stopMessage();
|
||||
stop.value = false;
|
||||
// playing.value = false;
|
||||
buildConnect();
|
||||
// cancelAnimationFrame(animationFrameId.value);
|
||||
} catch (err) {}
|
||||
}
|
||||
};
|
||||
// 每次call先上传当前用户配置
|
||||
const uploadUserConfig = async () => {
|
||||
if (!localStorage.getItem('configData')) {
|
||||
return new Promise(resolve => resolve());
|
||||
}
|
||||
const {
|
||||
videoQuality,
|
||||
useAudioPrompt,
|
||||
voiceClonePrompt,
|
||||
assistantPrompt,
|
||||
vadThreshold,
|
||||
audioFormat,
|
||||
base64Str
|
||||
} = JSON.parse(localStorage.getItem('configData'));
|
||||
const obj = {
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'input_audio',
|
||||
input_audio: {
|
||||
data: base64Str,
|
||||
format: audioFormat
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'options',
|
||||
options: {
|
||||
hd_video: videoQuality,
|
||||
use_audio_prompt: useAudioPrompt,
|
||||
vad_threshold: vadThreshold,
|
||||
voice_clone_prompt: voiceClonePrompt,
|
||||
assistant_prompt: assistantPrompt
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
};
|
||||
const { code, message, data } = await uploadConfig(obj);
|
||||
modelVersion.value = data?.choices?.content || '';
|
||||
return new Promise((resolve, reject) => {
|
||||
if (code !== 0) {
|
||||
ElMessage({
|
||||
type: 'error',
|
||||
message: message,
|
||||
duration: 3000,
|
||||
customClass: 'system-error'
|
||||
});
|
||||
reject();
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
};
|
||||
</script>
|
||||
<style lang="less" scoped>
|
||||
.voice-page {
|
||||
flex: 1;
|
||||
height: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
&-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 0 16px 16px;
|
||||
box-shadow: 0 0.5px 0 0 #e0e0e0;
|
||||
margin-bottom: 16px;
|
||||
.header-icon {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
img {
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
margin-right: 8px;
|
||||
}
|
||||
span {
|
||||
color: rgba(23, 23, 23, 0.9);
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 500;
|
||||
line-height: normal;
|
||||
margin-right: 40px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
}
|
||||
.voice-container {
|
||||
display: flex;
|
||||
.voice-icon {
|
||||
width: 191px;
|
||||
height: 45px;
|
||||
}
|
||||
}
|
||||
}
|
||||
&-output {
|
||||
flex: 1;
|
||||
height: 0;
|
||||
padding: 0 16px;
|
||||
margin-bottom: 16px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
.output-content {
|
||||
flex: 1;
|
||||
overflow: auto;
|
||||
}
|
||||
.skip-box {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: flex-end;
|
||||
margin-top: 16px;
|
||||
}
|
||||
}
|
||||
&-btn {
|
||||
text-align: center;
|
||||
padding: 8px 0;
|
||||
.el-button {
|
||||
width: 284px;
|
||||
height: 46px;
|
||||
border-radius: 8px;
|
||||
}
|
||||
.el-button.el-button--success {
|
||||
background: #647fff;
|
||||
border-color: #647fff;
|
||||
&:hover {
|
||||
opacity: 0.8;
|
||||
}
|
||||
span {
|
||||
color: #fff;
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 500;
|
||||
line-height: normal;
|
||||
}
|
||||
}
|
||||
.el-button.el-button--success.is-disabled {
|
||||
background: #f3f3f3;
|
||||
border-color: #f3f3f3;
|
||||
span {
|
||||
color: #d1d1d1;
|
||||
}
|
||||
}
|
||||
.el-button.el-button--danger {
|
||||
border-color: #dc3545;
|
||||
background-color: #dc3545;
|
||||
color: #ffffff;
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 500;
|
||||
line-height: normal;
|
||||
.phone-icon {
|
||||
margin-right: 10px;
|
||||
}
|
||||
.btn-text {
|
||||
margin-right: 10px;
|
||||
}
|
||||
.btn-desc {
|
||||
margin-right: 16px;
|
||||
}
|
||||
.time {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
.time-minute,
|
||||
.time-second {
|
||||
width: 26px;
|
||||
height: 26px;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
border-radius: 3.848px;
|
||||
background: rgba(47, 47, 47, 0.5);
|
||||
}
|
||||
.time-colon {
|
||||
margin: 0 3px;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,829 @@
|
||||
<template>
|
||||
<ExtraInfo webVersion="websocket_0107" :modelVersion="modelVersion" />
|
||||
<div class="voice-page">
|
||||
<div class="voice-page-header">
|
||||
<div class="header-icon">
|
||||
<img src="@/assets/images/voice-icon.png" />
|
||||
<span>Audio Choice</span>
|
||||
</div>
|
||||
<div class="voice-container" v-if="!isCalling">
|
||||
<SvgIcon name="voice" class="voice-icon" />
|
||||
<SvgIcon name="voice" class="voice-icon" />
|
||||
<SvgIcon name="voice" class="voice-icon" />
|
||||
</div>
|
||||
<div class="voice-container" v-else>
|
||||
<Voice
|
||||
:dataArray="dataArray"
|
||||
:isCalling="isCalling"
|
||||
:isPlaying="playing"
|
||||
:configList="videoConfigList"
|
||||
:boxStyle="{ height: '45px' }"
|
||||
:itemStyle="{ width: '3px', margin: '0 1px' }"
|
||||
/>
|
||||
</div>
|
||||
<!-- <SelectTimbre v-model:timbre="timbre" v-model:audioData="audioData" v-model:disabled="isCalling" /> -->
|
||||
</div>
|
||||
<div class="voice-page-output">
|
||||
<div class="output-content">
|
||||
<ModelOutput v-if="outputData.length > 0" :outputData="outputData" containerClass="output-content" />
|
||||
</div>
|
||||
<div class="skip-box">
|
||||
<DelayTips
|
||||
v-if="delayTimestamp > 200 || delayCount > 2"
|
||||
:delayTimestamp="delayTimestamp"
|
||||
:delayCount="delayCount"
|
||||
/>
|
||||
<LikeAndDislike v-model:feedbackStatus="feedbackStatus" v-model:curResponseId="curResponseId" />
|
||||
<SkipBtn :disabled="skipDisabled" @click="skipVoice" />
|
||||
</div>
|
||||
</div>
|
||||
<div class="voice-page-btn">
|
||||
<el-button v-show="!isCalling" type="success" :disabled="callDisabled" @click="initRecording">
|
||||
{{ callDisabled ? 'Not ready yet, please wait' : 'Call MiniCPM' }}
|
||||
</el-button>
|
||||
<el-button v-show="isCalling" @click="stopRecording" type="danger">
|
||||
<SvgIcon name="phone-icon" className="phone-icon" />
|
||||
<span class="btn-text">Hang Up</span>
|
||||
<CountDown v-model="isCalling" @timeUp="stopRecording" />
|
||||
</el-button>
|
||||
</div>
|
||||
<IdeasList v-if="showIdeasList" :ideasList="voiceIdeasList" />
|
||||
</div>
|
||||
</template>
|
||||
<script setup>
|
||||
import { sendMessage, stopMessage, uploadConfig } from '@/apis';
|
||||
import { encodeWAV } from '@/hooks/useVoice';
|
||||
import { getNewUserId, setNewUserId } from '@/hooks/useRandomId';
|
||||
import { fetchEventSource } from '@microsoft/fetch-event-source';
|
||||
import { MicVAD } from '@ricky0123/vad-web';
|
||||
import { videoConfigList, voiceConfigList, voiceIdeasList, showIdeasList } from '@/enums';
|
||||
import { getChunkLength } from '@/utils';
|
||||
import { mergeBase64ToBlob } from './merge';
|
||||
import WebSocketService from '@/utils/websocket';
|
||||
|
||||
let ctrl = new AbortController();
|
||||
let socket = null;
|
||||
const audioData = ref({
|
||||
base64Str: '',
|
||||
type: 'mp3'
|
||||
}); // 自定义音色base64
|
||||
const isCalling = defineModel();
|
||||
const taskQueue = ref([]);
|
||||
const running = ref(false);
|
||||
const outputData = ref([]);
|
||||
const textQueue = ref('');
|
||||
const textAnimationInterval = ref();
|
||||
|
||||
const isFirstReturn = ref(true); // 首次返回的音频是前端发给后端的音频片段，需要单独处理
|
||||
|
||||
const audioPlayQueue = ref([]);
|
||||
const base64List = ref([]);
|
||||
const playing = ref(false);
|
||||
const skipDisabled = ref(true);
|
||||
const stop = ref(false);
|
||||
const timbre = ref([1]);
|
||||
const isReturnError = ref(false);
|
||||
const allVoice = ref([]);
|
||||
const callDisabled = ref(true);
|
||||
|
||||
const feedbackStatus = ref('');
|
||||
const curResponseId = ref('');
|
||||
const delayTimestamp = ref(0); // 当前发送片延时
|
||||
const delayCount = ref(0); // 当前剩余多少ms未发送到接口
|
||||
|
||||
const modelVersion = ref('');
|
||||
|
||||
let audioDOM = new Audio();
|
||||
|
||||
const isEnd = ref(false); // sse接口关闭，认为模型已完成本次返回
|
||||
// 页面卸载时关闭录音
|
||||
onBeforeUnmount(() => {
|
||||
stopRecording();
|
||||
});
|
||||
const vadStartTime = ref();
|
||||
let myvad = null;
|
||||
let vadTimer = null; // vad定时器，用于检测1s内人声是否停止，1s内停止，可认为是vad误触，直接忽略，1s内未停止，则认为是人声，已自动跳过当前对话
|
||||
const vadStart = async () => {
|
||||
myvad = await MicVAD.new({
|
||||
onSpeechStart: () => {
|
||||
console.log('Speech start detected');
|
||||
if (!skipDisabled.value) {
|
||||
vadTimer && clearTimeout(vadTimer);
|
||||
vadTimer = setTimeout(() => {
|
||||
console.log('打断时间: ', +new Date());
|
||||
skipVoice();
|
||||
}, 500);
|
||||
}
|
||||
},
|
||||
onSpeechEnd: audio => {
|
||||
vadTimer && clearTimeout(vadTimer);
|
||||
// debugger;
|
||||
// do something with `audio` (Float32Array of audio samples at sample rate 16000)...
|
||||
}
|
||||
});
|
||||
console.log('vad: ', myvad);
|
||||
myvad.start();
|
||||
};
|
||||
onMounted(async () => {
|
||||
const { code, message } = await stopMessage();
|
||||
if (code !== 0) {
|
||||
ElMessage({
|
||||
type: 'error',
|
||||
message: message,
|
||||
duration: 3000,
|
||||
customClass: 'system-error'
|
||||
});
|
||||
return;
|
||||
}
|
||||
callDisabled.value = false;
|
||||
});
|
||||
const delay = ms => {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
};
|
||||
const initRecording = async () => {
|
||||
uploadUserConfig()
|
||||
.then(async () => {
|
||||
// 每次call都需要生成新uid
|
||||
setNewUserId();
|
||||
|
||||
outputData.value = [];
|
||||
buildConnect();
|
||||
isCalling.value = true;
|
||||
await delay(100);
|
||||
if (socket) {
|
||||
socket.close();
|
||||
}
|
||||
socket = new WebSocketService(
|
||||
`/ws/stream${window.location.search}&uid=${getNewUserId()}&service=minicpmo-server`
|
||||
);
|
||||
socket.connect();
|
||||
// 建立连接后稍等一会儿再传送数据
|
||||
startRecording();
|
||||
if (localStorage.getItem('canStopByVoice') === 'true') {
|
||||
vadStart();
|
||||
}
|
||||
})
|
||||
.catch(() => {});
|
||||
};
|
||||
let audioContext;
|
||||
const analyser = ref();
|
||||
const dataArray = ref();
|
||||
let mediaRecorder;
|
||||
let audioChunks = [];
|
||||
const animationFrameId = ref();
|
||||
|
||||
const isFirstPiece = ref(true);
|
||||
|
||||
const startRecording = async () => {
|
||||
// 获取用户音频流
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
|
||||
// 创建 AudioContext 和 MediaStreamAudioSourceNode
|
||||
audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 });
|
||||
const source = audioContext.createMediaStreamSource(stream);
|
||||
|
||||
analyser.value = audioContext.createAnalyser();
|
||||
// 将音频节点连接到分析器
|
||||
source.connect(analyser.value);
|
||||
// 分析器设置
|
||||
analyser.value.fftSize = 256;
|
||||
const bufferLength = analyser.value.frequencyBinCount;
|
||||
dataArray.value = new Uint8Array(bufferLength);
|
||||
// 开始绘制音波
|
||||
drawBars();
|
||||
|
||||
// 创建 ScriptProcessorNode 用于捕获音频数据
|
||||
const processor = audioContext.createScriptProcessor(256, 1, 1);
|
||||
|
||||
processor.onaudioprocess = event => {
|
||||
if (!isCalling.value) return;
|
||||
if (isReturnError.value) {
|
||||
stopRecording();
|
||||
return;
|
||||
}
|
||||
const data = event.inputBuffer.getChannelData(0);
|
||||
audioChunks.push(new Float32Array(data));
|
||||
// 检查是否已经收集到1秒钟的数据
|
||||
const totalBufferLength = audioChunks.reduce((total, curr) => total + curr.length, 0);
|
||||
const chunkLength = getChunkLength(audioContext.sampleRate);
|
||||
if (totalBufferLength >= chunkLength) {
|
||||
// 合并到一个完整的数据数组，并裁剪成1秒钟
|
||||
const mergedBuffer = mergeBuffers(audioChunks, totalBufferLength);
|
||||
const oneSecondBuffer = mergedBuffer.slice(0, chunkLength);
|
||||
// 保存并处理成WAV格式
|
||||
addQueue(+new Date(), () => saveAudioChunk(oneSecondBuffer, +new Date()));
|
||||
// 保留多余的数据备用
|
||||
audioChunks = [mergedBuffer.slice(chunkLength)];
|
||||
}
|
||||
};
|
||||
|
||||
source.connect(processor);
|
||||
processor.connect(audioContext.destination);
|
||||
};
|
||||
const stopRecording = () => {
|
||||
isCalling.value = false;
|
||||
if (animationFrameId.value) {
|
||||
cancelAnimationFrame(animationFrameId.value);
|
||||
}
|
||||
if (audioContext && audioContext.state !== 'closed') {
|
||||
audioContext.close();
|
||||
}
|
||||
ctrl.abort();
|
||||
ctrl = new AbortController();
|
||||
taskQueue.value = [];
|
||||
audioPlayQueue.value = [];
|
||||
base64List.value = [];
|
||||
isReturnError.value = false;
|
||||
skipDisabled.value = true;
|
||||
playing.value = false;
|
||||
audioDOM.pause();
|
||||
stopMessage();
|
||||
if (socket) {
|
||||
socket.close();
|
||||
}
|
||||
if (
|
||||
outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
|
||||
outputData.value[outputData.value.length - 1].audio === '' &&
|
||||
allVoice.value.length > 0
|
||||
) {
|
||||
outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
|
||||
}
|
||||
myvad && myvad.destroy();
|
||||
};
|
||||
const getStopValue = () => {
|
||||
return stop.value;
|
||||
};
|
||||
const getPlayingValue = () => {
|
||||
return playing.value;
|
||||
};
|
||||
const getStopStatus = () => {
|
||||
return localStorage.getItem('canStopByVoice') === 'true';
|
||||
};
|
||||
const saveAudioChunk = (buffer, timestamp) => {
|
||||
return new Promise(resolve => {
|
||||
if (!getStopStatus() && getPlayingValue()) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const wavBlob = encodeWAV(buffer, audioContext.sampleRate);
|
||||
let reader = new FileReader();
|
||||
reader.readAsDataURL(wavBlob);
|
||||
|
||||
reader.onloadend = async function () {
|
||||
let base64data = reader.result.split(',')[1];
|
||||
if (!base64data) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const obj = {
|
||||
uid: getNewUserId(),
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'input_audio',
|
||||
input_audio: {
|
||||
data: base64data,
|
||||
format: 'wav',
|
||||
timestamp: String(timestamp)
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
};
|
||||
socket.send(JSON.stringify(obj));
|
||||
socket.on('message', data => {
|
||||
console.log('message: ', data);
|
||||
delayTimestamp.value = +new Date() - timestamp;
|
||||
delayCount.value = taskQueue.value.length;
|
||||
resolve();
|
||||
});
|
||||
// 将Base64音频数据发送到后端
|
||||
// try {
|
||||
// await sendMessage(obj);
|
||||
// delayTimestamp.value = +new Date() - timestamp;
|
||||
// delayCount.value = taskQueue.value.length;
|
||||
// } catch (err) {}
|
||||
// resolve();
|
||||
};
|
||||
});
|
||||
};
|
||||
const mergeBuffers = (buffers, length) => {
|
||||
const result = new Float32Array(length);
|
||||
let offset = 0;
|
||||
for (let buffer of buffers) {
|
||||
result.set(buffer, offset);
|
||||
offset += buffer.length;
|
||||
}
|
||||
return result;
|
||||
};
|
||||
// 建立连接
|
||||
const buildConnect = async () => {
|
||||
const obj = {
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [{ type: 'none' }]
|
||||
}
|
||||
],
|
||||
stream: true
|
||||
};
|
||||
isEnd.value = false;
|
||||
ctrl.abort();
|
||||
ctrl = new AbortController();
|
||||
const url = `/api/v1/completions${window.location.search}`;
|
||||
fetchEventSource(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
service: 'minicpmo-server',
|
||||
uid: getNewUserId()
|
||||
},
|
||||
body: JSON.stringify(obj),
|
||||
signal: ctrl.signal,
|
||||
openWhenHidden: true,
|
||||
async onopen(response) {
|
||||
console.log('onopen', response);
|
||||
isFirstPiece.value = true;
|
||||
isFirstReturn.value = true;
|
||||
allVoice.value = [];
|
||||
base64List.value = [];
|
||||
if (response.status !== 200) {
|
||||
ElMessage({
|
||||
type: 'error',
|
||||
message: 'At limit. Please try again soon.',
|
||||
duration: 3000,
|
||||
customClass: 'system-error'
|
||||
});
|
||||
isReturnError.value = true;
|
||||
} else {
|
||||
isReturnError.value = false;
|
||||
// skipDisabled.value = false;
|
||||
drawText();
|
||||
}
|
||||
},
|
||||
onmessage(msg) {
|
||||
const data = JSON.parse(msg.data);
|
||||
if (data.response_id) {
|
||||
curResponseId.value = data.response_id;
|
||||
}
|
||||
if (data.choices[0]?.text) {
|
||||
textQueue.value += data.choices[0].text.replace('<end>', '');
|
||||
console.warn('text return time -------------------------------', +new Date());
|
||||
}
|
||||
// 首次返回的是前端发给后端的音频片段，需要单独处理
|
||||
if (isFirstReturn.value) {
|
||||
console.log('第一次');
|
||||
isFirstReturn.value = false;
|
||||
// 如果后端返回的音频为空，需要重连
|
||||
if (!data.choices[0].audio) {
|
||||
buildConnect();
|
||||
return;
|
||||
}
|
||||
outputData.value.push({
|
||||
type: 'USER',
|
||||
audio: `data:audio/wav;base64,${data.choices[0].audio}`
|
||||
});
|
||||
outputData.value.push({
|
||||
type: 'BOT',
|
||||
text: '',
|
||||
audio: ''
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (data.choices[0]?.audio) {
|
||||
console.warn('audio return time -------------------------------', +new Date());
|
||||
if (!getStopValue() && isCalling.value) {
|
||||
skipDisabled.value = false;
|
||||
base64List.value.push(`data:audio/wav;base64,${data.choices[0].audio}`);
|
||||
addAudioQueue(() => truePlay(data.choices[0].audio));
|
||||
}
|
||||
allVoice.value.push(`data:audio/wav;base64,${data.choices[0].audio}`);
|
||||
} else {
|
||||
// 发生异常了，直接重连
|
||||
buildConnect();
|
||||
}
|
||||
if (data.choices[0].text.includes('<end>')) {
|
||||
console.log('收到结束标记了:', +new Date());
|
||||
if (
|
||||
outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
|
||||
outputData.value[outputData.value.length - 1].audio === '' &&
|
||||
allVoice.value.length > 0
|
||||
) {
|
||||
outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
|
||||
}
|
||||
}
|
||||
},
|
||||
onclose() {
|
||||
console.log('onclose', +new Date());
|
||||
isEnd.value = true;
|
||||
outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
|
||||
vadStartTime.value = +new Date();
|
||||
if (audioPlayQueue.value.length === 0) {
|
||||
let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
|
||||
console.log('taskQueue111111111: ', taskQueue.value, startIndex);
|
||||
if (startIndex !== -1) {
|
||||
taskQueue.value = taskQueue.value.slice(startIndex);
|
||||
console.log('截取后长度:', taskQueue.value, vadStartTime.value);
|
||||
}
|
||||
buildConnect();
|
||||
}
|
||||
},
|
||||
onerror(err) {
|
||||
console.log('onerror', err);
|
||||
ctrl.abort();
|
||||
ctrl = new AbortController();
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
};
|
||||
const drawText = async () => {
|
||||
if (textQueue.value.length > 0) {
|
||||
outputData.value[outputData.value.length - 1].text += textQueue.value[0];
|
||||
textQueue.value = textQueue.value.slice(1);
|
||||
} else {
|
||||
cancelAnimationFrame(textAnimationInterval.value);
|
||||
}
|
||||
textAnimationInterval.value = requestAnimationFrame(drawText);
|
||||
};
|
||||
// 返回的语音放到队列里，挨个播放
|
||||
const addAudioQueue = async item => {
|
||||
audioPlayQueue.value.push(item);
|
||||
if (isFirstPiece.value) {
|
||||
await delay(500);
|
||||
isFirstPiece.value = false;
|
||||
}
|
||||
if (audioPlayQueue.value.length > 0 && !playing.value) {
|
||||
playing.value = true;
|
||||
playAudio();
|
||||
}
|
||||
};
|
||||
// 控制播放队列执行
|
||||
const playAudio = () => {
|
||||
console.log('剩余播放列表:', audioPlayQueue.value, +new Date());
|
||||
|
||||
if (!isEnd.value && base64List.value.length >= 2) {
|
||||
const remainLen = base64List.value.length;
|
||||
const blob = mergeBase64ToBlob(base64List.value);
|
||||
audioDOM.src = blob;
|
||||
audioDOM.play();
|
||||
console.error('前期合并后播放开始时间: ', +new Date());
|
||||
audioDOM.onended = () => {
|
||||
console.error('前期合并后播放结束时间: ', +new Date());
|
||||
base64List.value = base64List.value.slice(remainLen);
|
||||
audioPlayQueue.value = audioPlayQueue.value.slice(remainLen);
|
||||
playAudio();
|
||||
};
|
||||
return;
|
||||
}
|
||||
if (isEnd.value && base64List.value.length >= 2) {
|
||||
const blob = mergeBase64ToBlob(base64List.value);
|
||||
// let audio = new Audio();
|
||||
audioDOM.src = blob;
|
||||
audioDOM.play();
|
||||
console.error('最后合并后播放开始时间: ', +new Date());
|
||||
audioDOM.onended = () => {
|
||||
console.error('合并后播放结束时间: ', +new Date());
|
||||
// URL.revokeObjectURL(url);
|
||||
base64List.value = [];
|
||||
audioPlayQueue.value = [];
|
||||
playing.value = false;
|
||||
skipDisabled.value = true;
|
||||
if (isCalling.value && !isReturnError.value) {
|
||||
// skipDisabled.value = true;
|
||||
taskQueue.value = [];
|
||||
// 打断前记录一下打断时间或vad触发事件
|
||||
// vadStartTime.value = +new Date();
|
||||
// // 每次完成后只保留当前时刻往前推1s的语音
|
||||
// console.log(
|
||||
// '截取前长度:',
|
||||
// taskQueue.value.map(item => item.time)
|
||||
// );
|
||||
// let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
|
||||
// if (startIndex !== -1) {
|
||||
// taskQueue.value = taskQueue.value.slice(startIndex);
|
||||
// console.log(
|
||||
// '截取后长度:',
|
||||
// taskQueue.value.map(item => item.time),
|
||||
// vadStartTime.value
|
||||
// );
|
||||
// }
|
||||
buildConnect();
|
||||
}
|
||||
};
|
||||
return;
|
||||
}
|
||||
base64List.value.shift();
|
||||
const item = audioPlayQueue.value.shift();
|
||||
if (item) {
|
||||
item().finally(() => playAudio());
|
||||
} else {
|
||||
playing.value = false;
|
||||
if (isEnd.value) {
|
||||
console.warn('play done................');
|
||||
skipDisabled.value = true;
|
||||
}
|
||||
// 播放完成后且正在通话且接口未返回错误时开始下一次连接
|
||||
if (isEnd.value && isCalling.value && !isReturnError.value) {
|
||||
// skipDisabled.value = true;
|
||||
taskQueue.value = [];
|
||||
// 打断前记录一下打断时间或vad触发事件
|
||||
// vadStartTime.value = +new Date();
|
||||
// // 每次完成后只保留当前时刻往前推1s的语音
|
||||
// console.log(
|
||||
// '截取前长度:',
|
||||
// taskQueue.value.map(item => item.time)
|
||||
// );
|
||||
// let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
|
||||
// if (startIndex !== -1) {
|
||||
// taskQueue.value = taskQueue.value.slice(startIndex);
|
||||
// console.log(
|
||||
// '截取后长度:',
|
||||
// taskQueue.value.map(item => item.time),
|
||||
// vadStartTime.value
|
||||
// );
|
||||
// }
|
||||
buildConnect();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// 播放音频
|
||||
const truePlay = async voice => {
|
||||
return new Promise(resolve => {
|
||||
audioDOM.src = 'data:audio/wav;base64,' + voice;
|
||||
console.error('播放开始时间:', +new Date());
|
||||
audioDOM
|
||||
.play()
|
||||
.then(() => {
|
||||
// console.error('播放结束时间: ', +new Date());
|
||||
})
|
||||
.catch(error => {
|
||||
resolve();
|
||||
if (error.name === 'NotAllowedError' || error.name === 'SecurityError') {
|
||||
console.error('User interaction required or permission issue:', error);
|
||||
ElMessage.warning('音频播放失败');
|
||||
} else {
|
||||
console.error('Error playing audio:', error);
|
||||
}
|
||||
});
|
||||
|
||||
audioDOM.onended = () => {
|
||||
console.error('播放结束时间: ', +new Date());
|
||||
// URL.revokeObjectURL(url);
|
||||
resolve();
|
||||
};
|
||||
});
|
||||
};
|
||||
// 当队列中任务数大于0时，开始处理队列中的任务
|
||||
const addQueue = (time, item) => {
|
||||
taskQueue.value.push({ func: item, time });
|
||||
if (taskQueue.value.length > 0 && !running.value) {
|
||||
running.value = true;
|
||||
processQueue();
|
||||
}
|
||||
};
|
||||
const processQueue = () => {
|
||||
const item = taskQueue.value.shift();
|
||||
if (item?.func) {
|
||||
item.func().then(() => {
|
||||
console.warn('shift!!!!!!!!!');
|
||||
processQueue();
|
||||
});
|
||||
} else {
|
||||
running.value = false;
|
||||
}
|
||||
};
|
||||
const drawBars = () => {
|
||||
// AnalyserNode接口的 getByteFrequencyData() 方法将当前频率数据复制到传入的 Uint8Array（无符号字节数组）中。
|
||||
analyser.value.getByteFrequencyData(dataArray.value);
|
||||
animationFrameId.value = requestAnimationFrame(drawBars);
|
||||
};
|
||||
// 跳过当前片段
|
||||
const skipVoice = async () => {
|
||||
// 打断前记录一下打断时间或vad触发事件
|
||||
vadStartTime.value = +new Date();
|
||||
if (!skipDisabled.value) {
|
||||
if (
|
||||
outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
|
||||
outputData.value[outputData.value.length - 1].audio === ''
|
||||
) {
|
||||
outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
|
||||
}
|
||||
base64List.value = [];
|
||||
audioPlayQueue.value = [];
|
||||
// 跳过之后，只保留当前时间点两秒内到之后的音频片段
|
||||
console.log(
|
||||
'截取前长度:',
|
||||
taskQueue.value.map(item => item.time)
|
||||
);
|
||||
let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
|
||||
if (startIndex !== -1) {
|
||||
taskQueue.value = taskQueue.value.slice(startIndex);
|
||||
console.log(
|
||||
'截取后长度:',
|
||||
taskQueue.value.map(item => item.time),
|
||||
vadStartTime.value
|
||||
);
|
||||
}
|
||||
stop.value = true;
|
||||
audioDOM.pause();
|
||||
setTimeout(() => {
|
||||
skipDisabled.value = true;
|
||||
}, 300);
|
||||
try {
|
||||
playing.value = false;
|
||||
await stopMessage();
|
||||
stop.value = false;
|
||||
// playing.value = false;
|
||||
buildConnect();
|
||||
// cancelAnimationFrame(animationFrameId.value);
|
||||
} catch (err) {}
|
||||
}
|
||||
};
|
||||
// 每次call先上传当前用户配置
|
||||
const uploadUserConfig = async () => {
|
||||
if (!localStorage.getItem('configData')) {
|
||||
return new Promise(resolve => resolve());
|
||||
}
|
||||
const {
|
||||
videoQuality,
|
||||
useAudioPrompt,
|
||||
voiceClonePrompt,
|
||||
assistantPrompt,
|
||||
vadThreshold,
|
||||
audioFormat,
|
||||
base64Str
|
||||
} = JSON.parse(localStorage.getItem('configData'));
|
||||
const obj = {
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'input_audio',
|
||||
input_audio: {
|
||||
data: base64Str,
|
||||
format: audioFormat
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'options',
|
||||
options: {
|
||||
hd_video: videoQuality,
|
||||
use_audio_prompt: useAudioPrompt,
|
||||
vad_threshold: vadThreshold,
|
||||
voice_clone_prompt: voiceClonePrompt,
|
||||
assistant_prompt: assistantPrompt
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
};
|
||||
const { code, message, data } = await uploadConfig(obj);
|
||||
modelVersion.value = data?.choices?.content || '';
|
||||
return new Promise((resolve, reject) => {
|
||||
if (code !== 0) {
|
||||
ElMessage({
|
||||
type: 'error',
|
||||
message: message,
|
||||
duration: 3000,
|
||||
customClass: 'system-error'
|
||||
});
|
||||
reject();
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
};
|
||||
</script>
|
||||
<style lang="less">
|
||||
.voice-page {
|
||||
flex: 1;
|
||||
height: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
&-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
padding: 0 16px 16px;
|
||||
box-shadow: 0 0.5px 0 0 #e0e0e0;
|
||||
margin-bottom: 16px;
|
||||
justify-content: space-between;
|
||||
.header-icon {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
img {
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
margin-right: 8px;
|
||||
}
|
||||
span {
|
||||
color: rgba(23, 23, 23, 0.9);
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 500;
|
||||
line-height: normal;
|
||||
margin-right: 40px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
}
|
||||
.voice-container {
|
||||
display: flex;
|
||||
.voice-icon {
|
||||
width: 191px;
|
||||
height: 45px;
|
||||
}
|
||||
}
|
||||
}
|
||||
&-output {
|
||||
flex: 1;
|
||||
height: 0;
|
||||
padding: 0 16px;
|
||||
margin-bottom: 16px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
.output-content {
|
||||
flex: 1;
|
||||
overflow: auto;
|
||||
}
|
||||
.skip-box {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: flex-end;
|
||||
margin-top: 16px;
|
||||
}
|
||||
}
|
||||
&-btn {
|
||||
text-align: center;
|
||||
padding: 8px 0;
|
||||
.el-button {
|
||||
width: 284px;
|
||||
height: 46px;
|
||||
border-radius: 8px;
|
||||
}
|
||||
.el-button.el-button--success {
|
||||
background: #647fff;
|
||||
border-color: #647fff;
|
||||
&:hover {
|
||||
opacity: 0.8;
|
||||
}
|
||||
span {
|
||||
color: #fff;
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 500;
|
||||
line-height: normal;
|
||||
}
|
||||
}
|
||||
.el-button.el-button--success.is-disabled {
|
||||
background: #f3f3f3;
|
||||
border-color: #f3f3f3;
|
||||
span {
|
||||
color: #d1d1d1;
|
||||
}
|
||||
}
|
||||
.el-button.el-button--danger {
|
||||
border-color: #dc3545;
|
||||
background-color: #dc3545;
|
||||
color: #ffffff;
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 500;
|
||||
line-height: normal;
|
||||
.phone-icon {
|
||||
margin-right: 10px;
|
||||
}
|
||||
.btn-text {
|
||||
margin-right: 10px;
|
||||
}
|
||||
.btn-desc {
|
||||
margin-right: 16px;
|
||||
}
|
||||
.time {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
.time-minute,
|
||||
.time-second {
|
||||
width: 26px;
|
||||
height: 26px;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
border-radius: 3.848px;
|
||||
background: rgba(47, 47, 47, 0.5);
|
||||
}
|
||||
.time-colon {
|
||||
margin: 0 3px;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,36 @@
|
||||
import lame from '@breezystack/lamejs';
|
||||
|
||||
export const audioBufferToMp3Base64 = audioBuffer => {
|
||||
const mp3Encoder = new lame.Mp3Encoder(1, 16000, 128);
|
||||
const sampleBlockSize = 1152;
|
||||
const mp3Data = [];
|
||||
|
||||
for (let i = 0; i < audioBuffer.length; i += sampleBlockSize) {
|
||||
const sampleChunk = audioBuffer.subarray(i, i + sampleBlockSize);
|
||||
const mp3buf = mp3Encoder.encodeBuffer(sampleChunk);
|
||||
if (mp3buf.length > 0) {
|
||||
mp3Data.push(new Int8Array(mp3buf));
|
||||
}
|
||||
}
|
||||
|
||||
const mp3buf = mp3Encoder.flush();
|
||||
if (mp3buf.length > 0) {
|
||||
mp3Data.push(new Int8Array(mp3buf));
|
||||
}
|
||||
|
||||
const mp3Blob = new Blob(mp3Data, { type: 'audio/mp3' });
|
||||
const url = URL.createObjectURL(mp3Blob);
|
||||
let dom = document.querySelector('#voice-box');
|
||||
let audio = document.createElement('audio');
|
||||
audio.controls = true;
|
||||
audio.src = url;
|
||||
dom.appendChild(audio);
|
||||
return new Promise(resolve => {
|
||||
const reader = new FileReader();
|
||||
reader.onloadend = () => {
|
||||
const base64String = reader.result.split(',')[1];
|
||||
resolve(base64String);
|
||||
};
|
||||
reader.readAsDataURL(mp3Blob);
|
||||
});
|
||||
};
|
||||
@@ -0,0 +1,132 @@
|
||||
// Convert Base64 to ArrayBuffer
|
||||
const base64ToArrayBuffer = base64 => {
|
||||
const binaryString = atob(base64.split(',')[1]); // Remove data URI scheme if present
|
||||
const len = binaryString.length;
|
||||
const bytes = new Uint8Array(len);
|
||||
for (let i = 0; i < len; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
return bytes.buffer;
|
||||
};
|
||||
|
||||
// Parse WAV header and get audio data section
|
||||
const parseWav = buffer => {
|
||||
const view = new DataView(buffer);
|
||||
const format = view.getUint16(20, true);
|
||||
const channels = view.getUint16(22, true);
|
||||
const sampleRate = view.getUint32(24, true);
|
||||
const bitsPerSample = view.getUint16(34, true);
|
||||
const dataOffset = 44;
|
||||
const dataSize = view.getUint32(40, true);
|
||||
const audioData = new Uint8Array(buffer, dataOffset, dataSize);
|
||||
|
||||
return {
|
||||
format,
|
||||
channels,
|
||||
sampleRate,
|
||||
bitsPerSample,
|
||||
audioData
|
||||
};
|
||||
};
|
||||
|
||||
// Create WAV header for combined audio data
|
||||
const createWavHeader = (audioDataSize, sampleRate, channels, bitsPerSample) => {
|
||||
const arrayBuffer = new ArrayBuffer(44);
|
||||
const view = new DataView(arrayBuffer);
|
||||
|
||||
const writeString = (view, offset, string) => {
|
||||
for (let i = 0; i < string.length; i++) {
|
||||
view.setUint8(offset + i, string.charCodeAt(i));
|
||||
}
|
||||
};
|
||||
|
||||
writeString(view, 0, 'RIFF'); // ChunkID
|
||||
view.setUint32(4, 36 + audioDataSize, true); // ChunkSize
|
||||
writeString(view, 8, 'WAVE'); // Format
|
||||
writeString(view, 12, 'fmt '); // Subchunk1ID
|
||||
view.setUint32(16, 16, true); // Subchunk1Size (PCM)
|
||||
view.setUint16(20, 1, true); // AudioFormat (PCM)
|
||||
view.setUint16(22, channels, true); // NumChannels
|
||||
view.setUint32(24, sampleRate, true); // SampleRate
|
||||
view.setUint32(28, (sampleRate * channels * bitsPerSample) / 8, true); // ByteRate
|
||||
view.setUint16(32, (channels * bitsPerSample) / 8, true); // BlockAlign
|
||||
view.setUint16(34, bitsPerSample, true); // BitsPerSample
|
||||
writeString(view, 36, 'data'); // Subchunk2ID
|
||||
view.setUint32(40, audioDataSize, true); // Subchunk2Size
|
||||
|
||||
return arrayBuffer;
|
||||
};
|
||||
|
||||
// Merge multiple Base64 audio files and return a Blob
|
||||
const mergeAudioFiles = base64AudioArray => {
|
||||
let sampleRate, channels, bitsPerSample;
|
||||
let combinedAudioData = new Uint8Array();
|
||||
|
||||
for (let i = 0; i < base64AudioArray.length; i++) {
|
||||
const arrayBuffer = base64ToArrayBuffer(base64AudioArray[i]);
|
||||
const wav = parseWav(arrayBuffer);
|
||||
|
||||
// Initialize properties based on the first audio file
|
||||
if (i === 0) {
|
||||
sampleRate = wav.sampleRate;
|
||||
channels = wav.channels;
|
||||
bitsPerSample = wav.bitsPerSample;
|
||||
}
|
||||
|
||||
// Ensure all files have the same format
|
||||
if (wav.sampleRate !== sampleRate || wav.channels !== channels || wav.bitsPerSample !== bitsPerSample) {
|
||||
throw new Error('All audio files must have the same format.');
|
||||
}
|
||||
|
||||
// Combine audio data
|
||||
const newCombinedData = new Uint8Array(combinedAudioData.byteLength + wav.audioData.byteLength);
|
||||
newCombinedData.set(combinedAudioData, 0);
|
||||
newCombinedData.set(wav.audioData, combinedAudioData.byteLength);
|
||||
combinedAudioData = newCombinedData;
|
||||
}
|
||||
|
||||
const combinedAudioDataSize = combinedAudioData.byteLength;
|
||||
const wavHeader = createWavHeader(combinedAudioDataSize, sampleRate, channels, bitsPerSample);
|
||||
const combinedWavBuffer = new Uint8Array(wavHeader.byteLength + combinedAudioData.byteLength);
|
||||
combinedWavBuffer.set(new Uint8Array(wavHeader), 0);
|
||||
combinedWavBuffer.set(combinedAudioData, wavHeader.byteLength);
|
||||
|
||||
// Create a Blob from the combined audio data
|
||||
const combinedBlob = new Blob([combinedWavBuffer], { type: 'audio/wav' });
|
||||
return combinedBlob;
|
||||
};
|
||||
export const mergeBase64ToBlob = base64List => {
|
||||
const combinedBlob = mergeAudioFiles(base64List);
|
||||
const audioUrl = URL.createObjectURL(combinedBlob);
|
||||
return audioUrl;
|
||||
};
|
||||
|
||||
// 假设 base64Strings 是一个包含多个 Base64 编码 WAV 文件的数组
|
||||
// 注意：这些 Base64 字符串不应该包含 URI 前缀 (例如 "audio/wav;base64,")
|
||||
/**
|
||||
*
|
||||
* @param {Array} base64Strings
|
||||
* @returns
|
||||
*/
|
||||
// 解码 Base64 字符串并合并二进制数据
|
||||
export const mergeBase64WavFiles = base64Strings => {
|
||||
const binaryDataArray = base64Strings.map(base64 => {
|
||||
return Uint8Array.from(atob(base64), c => c.charCodeAt(0));
|
||||
});
|
||||
|
||||
const totalLength = binaryDataArray.reduce((sum, arr) => sum + arr.length, 0);
|
||||
|
||||
const mergedArray = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
|
||||
binaryDataArray.forEach(arr => {
|
||||
mergedArray.set(arr, offset);
|
||||
offset += arr.length;
|
||||
});
|
||||
|
||||
// 重新编码为 Base64 字符串
|
||||
const binaryString = String.fromCharCode(...mergedArray);
|
||||
const mergedBase64 = btoa(binaryString);
|
||||
|
||||
return mergedBase64;
|
||||
};
|
||||
@@ -0,0 +1,29 @@
|
||||
const base64ToArrayBuffer = base64 => {
|
||||
let binaryString = atob(base64);
|
||||
let len = binaryString.length;
|
||||
let bytes = new Uint8Array(len);
|
||||
for (let i = 0; i < len; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
return bytes.buffer;
|
||||
};
|
||||
|
||||
const concatenateArrayBuffers = buffers => {
|
||||
let totalLength = buffers.reduce((acc, value) => acc + value.byteLength, 0);
|
||||
let result = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
for (let buffer of buffers) {
|
||||
result.set(new Uint8Array(buffer), offset);
|
||||
offset += buffer.byteLength;
|
||||
}
|
||||
return result.buffer;
|
||||
};
|
||||
|
||||
export const mergeMp3Base64ToBlob = base64Strings => {
|
||||
let arrayBuffers = base64Strings.map(base64ToArrayBuffer);
|
||||
let combinedArrayBuffer = concatenateArrayBuffers(arrayBuffers);
|
||||
const blob = new Blob([combinedArrayBuffer], { type: 'audio/mp3' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
console.log('url', url);
|
||||
return url;
|
||||
};
|
||||
261
web_demos/minicpm-o_2.6/web_server/src/views/home/index.vue
Normal file
261
web_demos/minicpm-o_2.6/web_server/src/views/home/index.vue
Normal file
@@ -0,0 +1,261 @@
|
||||
<template>
|
||||
<div class="home-page">
|
||||
<div class="home-page-header">
|
||||
<div class="home-page-header-logo">
|
||||
<!-- <img src="@/assets/images/logo.png" /> -->
|
||||
<SvgIcon name="miniCPM2.6" class="logo-icon" />
|
||||
</div>
|
||||
<div class="home-page-header-menu">
|
||||
<div
|
||||
class="home-page-header-menu-item"
|
||||
v-for="(item, index) in tabList"
|
||||
:key="item.type"
|
||||
:class="`home-page-header-menu-item ${activeTab === item.type ? 'active-tab' : ''} ${item.disabled ? 'disabled-tab' : ''}`"
|
||||
@click="handleClickTab(item.type, index)"
|
||||
>
|
||||
{{ getMenuTab(item.type) }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="home-page-header-switch">
|
||||
<div class="change-language">
|
||||
<div
|
||||
:class="`change-language-item ${language === 'en' ? 'active-language' : ''}`"
|
||||
@click="handleChangeLanguage('en')"
|
||||
>
|
||||
English
|
||||
</div>
|
||||
<div
|
||||
:class="`change-language-item ${language === 'zh' ? 'active-language' : ''}`"
|
||||
@click="handleChangeLanguage('zh')"
|
||||
>
|
||||
中文
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div :class="`home-page-content ${activeTab === 'chatbot' && 'no-padding'}`">
|
||||
<VoiceCallWs v-if="isWebSocket && activeTab === 'voice'" v-model="isCalling" />
|
||||
<VoiceCall v-else-if="!isWebSocket && activeTab === 'voice'" v-model="isCalling" />
|
||||
<VideoCallWs v-else-if="isWebSocket && activeTab === 'video'" v-model="isCalling" />
|
||||
<VideoCall v-else-if="!isWebSocket && activeTab === 'video'" v-model="isCalling" />
|
||||
<iframe
|
||||
src="https://minicpm-omni-webdemo-iframe.modelbest.cn"
|
||||
frameborder="0"
|
||||
width="100%"
|
||||
height="100%"
|
||||
v-else
|
||||
/>
|
||||
<div class="config-box" v-if="activeTab !== 'chatbot'">
|
||||
<ModelConfig v-model:isCalling="isCalling" v-model:type="activeTab" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import VoiceCall from './components/VoiceCall.vue';
|
||||
import VoiceCallWs from './components/VoiceCall_0105.vue';
|
||||
import VideoCall from './components/VideoCall.vue';
|
||||
import VideoCallWs from './components/VideoCall_0105.vue';
|
||||
import { useI18n } from 'vue-i18n';
|
||||
import { useRoute, useRouter } from 'vue-router';
|
||||
|
||||
const route = useRoute();
|
||||
const router = useRouter();
|
||||
const typeObj = {
|
||||
0: 'video',
|
||||
1: 'voice',
|
||||
2: 'chatbot'
|
||||
};
|
||||
const defaultType = typeObj[route.query.type] || 'voice';
|
||||
|
||||
const { t, locale } = useI18n();
|
||||
const activeTab = ref(defaultType);
|
||||
const language = ref(localStorage.getItem('language') || 'zh');
|
||||
const isWebSocket = false;
|
||||
const tabList = ref([
|
||||
{
|
||||
type: 'video',
|
||||
text: 'Realtime Video Call'
|
||||
},
|
||||
{
|
||||
type: 'voice',
|
||||
text: 'Realtime Voice Call'
|
||||
},
|
||||
{
|
||||
type: 'chatbot',
|
||||
text: 'Chatbot'
|
||||
// disabled: true
|
||||
}
|
||||
]);
|
||||
const isCalling = ref(false);
|
||||
const handleChangeLanguage = val => {
|
||||
console.log('val: ', val);
|
||||
language.value = val;
|
||||
locale.value = val;
|
||||
localStorage.setItem('language', val);
|
||||
};
|
||||
const getMenuTab = val => {
|
||||
let text = '';
|
||||
switch (val) {
|
||||
case 'video':
|
||||
text = t('menuTabVideo');
|
||||
break;
|
||||
case 'voice':
|
||||
text = t('menuTabAudio');
|
||||
break;
|
||||
case 'chatbot':
|
||||
text = t('menuTabChatbot');
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return text;
|
||||
};
|
||||
const handleClickTab = (val, index) => {
|
||||
activeTab.value = val;
|
||||
const port = route.query.port;
|
||||
const type = index;
|
||||
router.push({
|
||||
path: '/',
|
||||
query: {
|
||||
port,
|
||||
type
|
||||
}
|
||||
});
|
||||
};
|
||||
</script>
|
||||
|
||||
<style lang="less" scoped>
|
||||
.home-page {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
&-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
&-logo {
|
||||
width: 174px;
|
||||
height: 46px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
border-radius: 12px;
|
||||
background: #ffffff;
|
||||
flex-shrink: 0;
|
||||
padding: 0 24px;
|
||||
.logo-icon {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
}
|
||||
&-menu {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
margin-left: 16px;
|
||||
&-item {
|
||||
width: 260px;
|
||||
height: 46px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
background: #ffffff;
|
||||
color: #252525;
|
||||
font-family: PingFang SC;
|
||||
font-size: 16px;
|
||||
font-style: normal;
|
||||
font-weight: 400;
|
||||
line-height: normal;
|
||||
border: 1px solid #dde1eb;
|
||||
cursor: pointer;
|
||||
user-select: none;
|
||||
}
|
||||
&-item + &-item {
|
||||
border-left: none;
|
||||
}
|
||||
&-item:first-of-type {
|
||||
border-radius: 12px 0 0 12px;
|
||||
}
|
||||
&-item:last-of-type {
|
||||
border-radius: 0 12px 12px 0;
|
||||
}
|
||||
.active-tab {
|
||||
color: #ffffff;
|
||||
background: linear-gradient(90deg, #789efe 0.02%, #647fff 75.28%);
|
||||
font-weight: 500;
|
||||
}
|
||||
.disabled-tab {
|
||||
cursor: not-allowed;
|
||||
border-color: #dde1eb;
|
||||
color: #d1d1d1;
|
||||
}
|
||||
}
|
||||
&-switch {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: flex-end;
|
||||
.change-language {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
&-item {
|
||||
width: 80px;
|
||||
height: 32px;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
border: 1px solid #dde1eb;
|
||||
background: #ffffff;
|
||||
color: #252525;
|
||||
font-family: PingFang SC;
|
||||
font-size: 14px;
|
||||
font-weight: 400;
|
||||
line-height: normal;
|
||||
cursor: pointer;
|
||||
user-select: none;
|
||||
}
|
||||
&-item:first-of-type {
|
||||
border-right: none;
|
||||
border-radius: 12px 0 0 12px;
|
||||
}
|
||||
&-item:last-of-type {
|
||||
border-radius: 0 12px 12px 0;
|
||||
}
|
||||
&-item.active-language {
|
||||
color: #ffffff;
|
||||
background: linear-gradient(90deg, #789efe 0.02%, #647fff 75.28%);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
&-content {
|
||||
flex: 1;
|
||||
height: 0;
|
||||
border-radius: 12px;
|
||||
margin-top: 16px;
|
||||
background: #ffffff;
|
||||
padding: 18px;
|
||||
display: flex;
|
||||
.config-box {
|
||||
width: 322px;
|
||||
margin-left: 16px;
|
||||
// border-left: 1px solid black;
|
||||
box-shadow: -0.5px 0 0 0 #e0e0e0;
|
||||
overflow: auto;
|
||||
}
|
||||
}
|
||||
.no-padding {
|
||||
padding: 0;
|
||||
overflow: hidden;
|
||||
background: #ffffff;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
<style lang="less">
|
||||
.el-popover.el-popper.config-popover {
|
||||
padding: 18px;
|
||||
border-radius: 12px;
|
||||
}
|
||||
</style>
|
||||
Reference in New Issue
Block a user