mirror of
https://github.com/HumanAIGC/lite-avatar.git
synced 2026-02-05 09:59:18 +08:00
add files
This commit is contained in:
烨玮
67
funasr_local/runtime/python/websocket/README.md
Normal file
67
funasr_local/runtime/python/websocket/README.md
Normal file
@@ -0,0 +1,67 @@
|
||||
# Service with websocket-python
|
||||
|
||||
This is a demo using funasr pipeline with websocket python-api.
|
||||
|
||||
## For the Server
|
||||
|
||||
### Install the modelscope and funasr
|
||||
|
||||
```shell
|
||||
pip install -U modelscope funasr
|
||||
# For the users in China, you could install with the command:
|
||||
# pip install -U modelscope funasr -i https://mirror.sjtu.edu.cn/pypi/web/simple
|
||||
git clone https://github.com/alibaba/FunASR.git && cd FunASR
|
||||
```
|
||||
|
||||
### Install the requirements for server
|
||||
|
||||
```shell
|
||||
cd funasr/runtime/python/websocket
|
||||
pip install -r requirements_server.txt
|
||||
```
|
||||
|
||||
### Start server
|
||||
#### ASR offline server
|
||||
|
||||
[//]: # (```shell)
|
||||
|
||||
[//]: # (python ws_server_online.py --host "0.0.0.0" --port 10095 --asr_model "damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch")
|
||||
|
||||
[//]: # (```)
|
||||
#### ASR streaming server
|
||||
```shell
|
||||
python ws_server_online.py --host "0.0.0.0" --port 10095 --asr_model_online "damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online"
|
||||
```
|
||||
|
||||
#### ASR offline/online 2pass server
|
||||
|
||||
[//]: # (```shell)
|
||||
|
||||
[//]: # (python ws_server_online.py --host "0.0.0.0" --port 10095 --asr_model "damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch")
|
||||
|
||||
[//]: # (```)
|
||||
|
||||
## For the client
|
||||
|
||||
Install the requirements for client
|
||||
```shell
|
||||
git clone https://github.com/alibaba/FunASR.git && cd FunASR
|
||||
cd funasr/runtime/python/websocket
|
||||
pip install -r requirements_client.txt
|
||||
```
|
||||
|
||||
### Start client
|
||||
#### Recording from mircrophone
|
||||
```shell
|
||||
# --chunk_size, "5,10,5"=600ms, "8,8,4"=480ms
|
||||
python ws_client.py --host "127.0.0.1" --port 10095 --chunk_size "5,10,5"
|
||||
```
|
||||
#### Loadding from wav.scp(kaldi style)
|
||||
```shell
|
||||
# --chunk_size, "5,10,5"=600ms, "8,8,4"=480ms
|
||||
python ws_client.py --host "127.0.0.1" --port 10095 --chunk_size "5,10,5" --audio_in "./data/wav.scp"
|
||||
```
|
||||
|
||||
## Acknowledge
|
||||
1. This project is maintained by [FunASR community](https://github.com/alibaba-damo-academy/FunASR).
|
||||
2. We acknowledge [cgisky1980](https://github.com/cgisky1980/FunASR) for contributing the websocket service.
|
||||
35
funasr_local/runtime/python/websocket/parse_args.py
Normal file
35
funasr_local/runtime/python/websocket/parse_args.py
Normal file
@@ -0,0 +1,35 @@
|
||||
# -*- encoding: utf-8 -*-
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--host",
|
||||
type=str,
|
||||
default="0.0.0.0",
|
||||
required=False,
|
||||
help="host ip, localhost, 0.0.0.0")
|
||||
parser.add_argument("--port",
|
||||
type=int,
|
||||
default=10095,
|
||||
required=False,
|
||||
help="grpc server port")
|
||||
parser.add_argument("--asr_model",
|
||||
type=str,
|
||||
default="damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
|
||||
help="model from modelscope")
|
||||
parser.add_argument("--asr_model_online",
|
||||
type=str,
|
||||
default="damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online",
|
||||
help="model from modelscope")
|
||||
parser.add_argument("--vad_model",
|
||||
type=str,
|
||||
default="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch",
|
||||
help="model from modelscope")
|
||||
parser.add_argument("--punc_model",
|
||||
type=str,
|
||||
default="damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727",
|
||||
help="model from modelscope")
|
||||
parser.add_argument("--ngpu",
|
||||
type=int,
|
||||
default=1,
|
||||
help="0 for cpu, 1 for gpu")
|
||||
|
||||
args = parser.parse_args()
|
||||
@@ -0,0 +1,2 @@
|
||||
websockets
|
||||
pyaudio
|
||||
@@ -0,0 +1 @@
|
||||
websockets
|
||||
182
funasr_local/runtime/python/websocket/ws_client.py
Normal file
182
funasr_local/runtime/python/websocket/ws_client.py
Normal file
@@ -0,0 +1,182 @@
|
||||
# -*- encoding: utf-8 -*-
|
||||
import os
|
||||
import time
|
||||
import websockets
|
||||
import asyncio
|
||||
# import threading
|
||||
import argparse
|
||||
import json
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--host",
|
||||
type=str,
|
||||
default="localhost",
|
||||
required=False,
|
||||
help="host ip, localhost, 0.0.0.0")
|
||||
parser.add_argument("--port",
|
||||
type=int,
|
||||
default=10095,
|
||||
required=False,
|
||||
help="grpc server port")
|
||||
parser.add_argument("--chunk_size",
|
||||
type=str,
|
||||
default="5, 10, 5",
|
||||
help="chunk")
|
||||
parser.add_argument("--chunk_interval",
|
||||
type=int,
|
||||
default=10,
|
||||
help="chunk")
|
||||
parser.add_argument("--audio_in",
|
||||
type=str,
|
||||
default=None,
|
||||
help="audio_in")
|
||||
|
||||
args = parser.parse_args()
|
||||
args.chunk_size = [int(x) for x in args.chunk_size.split(",")]
|
||||
|
||||
# voices = asyncio.Queue()
|
||||
from queue import Queue
|
||||
voices = Queue()
|
||||
|
||||
# 其他函数可以通过调用send(data)来发送数据，例如：
|
||||
async def record_microphone():
|
||||
is_finished = False
|
||||
import pyaudio
|
||||
#print("2")
|
||||
global voices
|
||||
FORMAT = pyaudio.paInt16
|
||||
CHANNELS = 1
|
||||
RATE = 16000
|
||||
chunk_size = 60*args.chunk_size[1]/args.chunk_interval
|
||||
CHUNK = int(RATE / 1000 * chunk_size)
|
||||
|
||||
p = pyaudio.PyAudio()
|
||||
|
||||
stream = p.open(format=FORMAT,
|
||||
channels=CHANNELS,
|
||||
rate=RATE,
|
||||
input=True,
|
||||
frames_per_buffer=CHUNK)
|
||||
is_speaking = True
|
||||
while True:
|
||||
|
||||
data = stream.read(CHUNK)
|
||||
data = data.decode('ISO-8859-1')
|
||||
message = json.dumps({"chunk_size": args.chunk_size, "chunk_interval": args.chunk_interval, "audio": data, "is_speaking": is_speaking, "is_finished": is_finished})
|
||||
|
||||
voices.put(message)
|
||||
#print(voices.qsize())
|
||||
|
||||
await asyncio.sleep(0.005)
|
||||
|
||||
# 其他函数可以通过调用send(data)来发送数据，例如：
|
||||
async def record_from_scp():
|
||||
import wave
|
||||
global voices
|
||||
is_finished = False
|
||||
if args.audio_in.endswith(".scp"):
|
||||
f_scp = open(args.audio_in)
|
||||
wavs = f_scp.readlines()
|
||||
else:
|
||||
wavs = [args.audio_in]
|
||||
for wav in wavs:
|
||||
wav_splits = wav.strip().split()
|
||||
wav_path = wav_splits[1] if len(wav_splits) > 1 else wav_splits[0]
|
||||
# bytes_f = open(wav_path, "rb")
|
||||
# bytes_data = bytes_f.read()
|
||||
with wave.open(wav_path, "rb") as wav_file:
|
||||
# 获取音频参数
|
||||
params = wav_file.getparams()
|
||||
# 获取头信息的长度
|
||||
# header_length = wav_file.getheaders()[0][1]
|
||||
# 读取音频帧数据，跳过头信息
|
||||
# wav_file.setpos(header_length)
|
||||
frames = wav_file.readframes(wav_file.getnframes())
|
||||
|
||||
# 将音频帧数据转换为字节类型的数据
|
||||
audio_bytes = bytes(frames)
|
||||
# stride = int(args.chunk_size/1000*16000*2)
|
||||
stride = int(60*args.chunk_size[1]/args.chunk_interval/1000*16000*2)
|
||||
chunk_num = (len(audio_bytes)-1)//stride + 1
|
||||
# print(stride)
|
||||
is_speaking = True
|
||||
for i in range(chunk_num):
|
||||
if i == chunk_num-1:
|
||||
is_speaking = False
|
||||
beg = i*stride
|
||||
data = audio_bytes[beg:beg+stride]
|
||||
data = data.decode('ISO-8859-1')
|
||||
message = json.dumps({"chunk_size": args.chunk_size, "chunk_interval": args.chunk_interval, "is_speaking": is_speaking, "audio": data, "is_finished": is_finished})
|
||||
voices.put(message)
|
||||
# print("data_chunk: ", len(data_chunk))
|
||||
# print(voices.qsize())
|
||||
|
||||
await asyncio.sleep(60*args.chunk_size[1]/args.chunk_interval/1000)
|
||||
|
||||
is_finished = True
|
||||
message = json.dumps({"is_finished": is_finished})
|
||||
voices.put(message)
|
||||
|
||||
async def ws_send():
|
||||
global voices
|
||||
global websocket
|
||||
print("started to sending data!")
|
||||
while True:
|
||||
while not voices.empty():
|
||||
data = voices.get()
|
||||
voices.task_done()
|
||||
try:
|
||||
await websocket.send(data) # 通过ws对象发送数据
|
||||
except Exception as e:
|
||||
print('Exception occurred:', e)
|
||||
await asyncio.sleep(0.005)
|
||||
await asyncio.sleep(0.005)
|
||||
|
||||
|
||||
|
||||
async def message():
|
||||
global websocket
|
||||
text_print = ""
|
||||
while True:
|
||||
try:
|
||||
meg = await websocket.recv()
|
||||
meg = json.loads(meg)
|
||||
# print(meg, end = '')
|
||||
# print("\r")
|
||||
text = meg["text"][0]
|
||||
text_print += text
|
||||
text_print = text_print[-55:]
|
||||
os.system('clear')
|
||||
print("\r"+text_print)
|
||||
except Exception as e:
|
||||
print("Exception:", e)
|
||||
|
||||
|
||||
async def print_messge():
|
||||
global websocket
|
||||
while True:
|
||||
try:
|
||||
meg = await websocket.recv()
|
||||
meg = json.loads(meg)
|
||||
print(meg)
|
||||
except Exception as e:
|
||||
print("Exception:", e)
|
||||
|
||||
|
||||
async def ws_client():
|
||||
global websocket # 定义一个全局变量ws，用于保存websocket连接对象
|
||||
# uri = "ws://11.167.134.197:8899"
|
||||
uri = "ws://{}:{}".format(args.host, args.port)
|
||||
#ws = await websockets.connect(uri, subprotocols=["binary"]) # 创建一个长连接
|
||||
async for websocket in websockets.connect(uri, subprotocols=["binary"], ping_interval=None):
|
||||
if args.audio_in is not None:
|
||||
task = asyncio.create_task(record_from_scp()) # 创建一个后台任务录音
|
||||
else:
|
||||
task = asyncio.create_task(record_microphone()) # 创建一个后台任务录音
|
||||
task2 = asyncio.create_task(ws_send()) # 创建一个后台任务发送
|
||||
task3 = asyncio.create_task(message()) # 创建一个后台接收消息的任务
|
||||
await asyncio.gather(task, task2, task3)
|
||||
|
||||
|
||||
asyncio.get_event_loop().run_until_complete(ws_client()) # 启动协程
|
||||
asyncio.get_event_loop().run_forever()
|
||||
108
funasr_local/runtime/python/websocket/ws_server_online.py
Normal file
108
funasr_local/runtime/python/websocket/ws_server_online.py
Normal file
@@ -0,0 +1,108 @@
|
||||
import asyncio
|
||||
import json
|
||||
import websockets
|
||||
import time
|
||||
from queue import Queue
|
||||
import threading
|
||||
import logging
|
||||
import tracemalloc
|
||||
import numpy as np
|
||||
|
||||
from parse_args import args
|
||||
from modelscope.pipelines import pipeline
|
||||
from modelscope.utils.constant import Tasks
|
||||
from modelscope.utils.logger import get_logger
|
||||
from funasr_local_onnx.utils.frontend import load_bytes
|
||||
|
||||
tracemalloc.start()
|
||||
|
||||
logger = get_logger(log_level=logging.CRITICAL)
|
||||
logger.setLevel(logging.CRITICAL)
|
||||
|
||||
|
||||
websocket_users = set()
|
||||
|
||||
|
||||
print("model loading")
|
||||
|
||||
inference_pipeline_asr_online = pipeline(
|
||||
task=Tasks.auto_speech_recognition,
|
||||
model=args.asr_model_online,
|
||||
model_revision='v1.0.4')
|
||||
|
||||
print("model loaded")
|
||||
|
||||
|
||||
|
||||
async def ws_serve(websocket, path):
|
||||
frames_online = []
|
||||
global websocket_users
|
||||
websocket.send_msg = Queue()
|
||||
websocket_users.add(websocket)
|
||||
websocket.param_dict_asr_online = {"cache": dict()}
|
||||
websocket.speek_online = Queue()
|
||||
ss_online = threading.Thread(target=asr_online, args=(websocket,))
|
||||
ss_online.start()
|
||||
|
||||
try:
|
||||
async for message in websocket:
|
||||
message = json.loads(message)
|
||||
is_finished = message["is_finished"]
|
||||
if not is_finished:
|
||||
audio = bytes(message['audio'], 'ISO-8859-1')
|
||||
|
||||
is_speaking = message["is_speaking"]
|
||||
websocket.param_dict_asr_online["is_final"] = not is_speaking
|
||||
|
||||
websocket.param_dict_asr_online["chunk_size"] = message["chunk_size"]
|
||||
|
||||
|
||||
frames_online.append(audio)
|
||||
|
||||
if len(frames_online) % message["chunk_interval"] == 0 or not is_speaking:
|
||||
|
||||
audio_in = b"".join(frames_online)
|
||||
websocket.speek_online.put(audio_in)
|
||||
frames_online = []
|
||||
|
||||
if not websocket.send_msg.empty():
|
||||
await websocket.send(websocket.send_msg.get())
|
||||
websocket.send_msg.task_done()
|
||||
|
||||
|
||||
except websockets.ConnectionClosed:
|
||||
print("ConnectionClosed...", websocket_users) # 链接断开
|
||||
websocket_users.remove(websocket)
|
||||
except websockets.InvalidState:
|
||||
print("InvalidState...") # 无效状态
|
||||
except Exception as e:
|
||||
print("Exception:", e)
|
||||
|
||||
|
||||
|
||||
def asr_online(websocket): # ASR推理
|
||||
global websocket_users
|
||||
while websocket in websocket_users:
|
||||
if not websocket.speek_online.empty():
|
||||
audio_in = websocket.speek_online.get()
|
||||
websocket.speek_online.task_done()
|
||||
if len(audio_in) > 0:
|
||||
# print(len(audio_in))
|
||||
audio_in = load_bytes(audio_in)
|
||||
rec_result = inference_pipeline_asr_online(audio_in=audio_in,
|
||||
param_dict=websocket.param_dict_asr_online)
|
||||
if websocket.param_dict_asr_online["is_final"]:
|
||||
websocket.param_dict_asr_online["cache"] = dict()
|
||||
|
||||
if "text" in rec_result:
|
||||
if rec_result["text"] != "sil" and rec_result["text"] != "waiting_for_more_voice":
|
||||
print(rec_result["text"])
|
||||
message = json.dumps({"mode": "online", "text": rec_result["text"]})
|
||||
websocket.send_msg.put(message)
|
||||
|
||||
time.sleep(0.005)
|
||||
|
||||
|
||||
start_server = websockets.serve(ws_serve, args.host, args.port, subprotocols=["binary"], ping_interval=None)
|
||||
asyncio.get_event_loop().run_until_complete(start_server)
|
||||
asyncio.get_event_loop().run_forever()
|
||||
Reference in New Issue
Block a user