mirror of
https://github.com/FunAudioLLM/CosyVoice.git
synced 2026-02-05 18:09:24 +08:00
add streaming dit
This commit is contained in:
yuekaiz
@@ -1,6 +1,6 @@
|
||||
#!/bin/bash
|
||||
# Copyright (c) 2025 NVIDIA (authors: Yuekai Zhang)
|
||||
export CUDA_VISIBLE_DEVICES=0
|
||||
export CUDA_VISIBLE_DEVICES=1
|
||||
cosyvoice_path=/workspace/CosyVoice
|
||||
cosyvoice_path=/workspace_yuekai/tts/CosyVoice
|
||||
stepaudio2_path=/workspace_yuekai/tts/Step-Audio2
|
||||
@@ -16,7 +16,7 @@ trt_dtype=bfloat16
|
||||
trt_weights_dir=./trt_weights_${trt_dtype}
|
||||
trt_engines_dir=./trt_engines_${trt_dtype}
|
||||
|
||||
model_repo=./model_repo_cosyvoice2
|
||||
model_repo=./model_repo_cosyvoice2_dit
|
||||
|
||||
use_spk2info_cache=False
|
||||
|
||||
@@ -58,40 +58,78 @@ if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then
|
||||
--engine_dir=$trt_engines_dir || exit 1
|
||||
fi
|
||||
|
||||
|
||||
# if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then
|
||||
# echo "Creating model repository"
|
||||
# rm -rf $model_repo
|
||||
# mkdir -p $model_repo
|
||||
# cosyvoice2_dir="cosyvoice2_dit"
|
||||
# token2wav_dir="token2wav_dit"
|
||||
|
||||
# cp -r ./model_repo/${cosyvoice2_dir} $model_repo
|
||||
# cp -r ./model_repo/tensorrt_llm $model_repo
|
||||
# cp -r ./model_repo/${token2wav_dir} $model_repo
|
||||
# #if [ $use_spk2info_cache == "False" ]; then
|
||||
# cp -r ./model_repo/audio_tokenizer $model_repo
|
||||
# cp -r ./model_repo/speaker_embedding $model_repo
|
||||
# #fi
|
||||
|
||||
# ENGINE_PATH=$trt_engines_dir
|
||||
# MAX_QUEUE_DELAY_MICROSECONDS=0
|
||||
# MODEL_DIR=$model_scope_model_local_dir
|
||||
# LLM_TOKENIZER_DIR=$huggingface_model_local_dir
|
||||
# BLS_INSTANCE_NUM=1
|
||||
# TRITON_MAX_BATCH_SIZE=16
|
||||
# DECOUPLED_MODE=True # True for streaming, False for offline
|
||||
# STEP_AUDIO_MODEL_DIR=/workspace_yuekai/tts/CosyVoice/runtime/triton_trtllm/Step-Audio-2-mini/token2wav
|
||||
|
||||
# python3 scripts/fill_template.py -i ${model_repo}/${token2wav_dir}/config.pbtxt model_dir:${STEP_AUDIO_MODEL_DIR},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS}
|
||||
# python3 scripts/fill_template.py -i ${model_repo}/${cosyvoice2_dir}/config.pbtxt model_dir:${MODEL_DIR},bls_instance_num:${BLS_INSTANCE_NUM},llm_tokenizer_dir:${LLM_TOKENIZER_DIR},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},decoupled_mode:${DECOUPLED_MODE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS}
|
||||
# python3 scripts/fill_template.py -i ${model_repo}/tensorrt_llm/config.pbtxt triton_backend:tensorrtllm,triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},decoupled_mode:${DECOUPLED_MODE},max_beam_width:1,engine_dir:${ENGINE_PATH},max_tokens_in_paged_kv_cache:2560,max_attention_window_size:2560,kv_cache_free_gpu_mem_fraction:0.5,exclude_input_in_output:True,enable_kv_cache_reuse:False,batching_strategy:inflight_fused_batching,max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS},encoder_input_features_data_type:TYPE_FP16,logits_datatype:TYPE_FP32
|
||||
# #if [ $use_spk2info_cache == "False" ]; then
|
||||
# python3 scripts/fill_template.py -i ${model_repo}/audio_tokenizer/config.pbtxt model_dir:${MODEL_DIR},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS}
|
||||
# python3 scripts/fill_template.py -i ${model_repo}/speaker_embedding/config.pbtxt model_dir:${MODEL_DIR},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS}
|
||||
# #fi
|
||||
# fi
|
||||
|
||||
if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then
|
||||
echo "Creating model repository"
|
||||
echo "Creating model repository async mode"
|
||||
rm -rf $model_repo
|
||||
mkdir -p $model_repo
|
||||
cosyvoice2_dir="cosyvoice2"
|
||||
cosyvoice2_dir="cosyvoice2_dit"
|
||||
token2wav_dir="token2wav_dit"
|
||||
|
||||
cp -r ./model_repo/${cosyvoice2_dir} $model_repo
|
||||
cp -r ./model_repo/tensorrt_llm $model_repo
|
||||
cp -r ./model_repo/token2wav $model_repo
|
||||
if [ $use_spk2info_cache == "False" ]; then
|
||||
cp -r ./model_repo/${token2wav_dir} $model_repo
|
||||
#if [ $use_spk2info_cache == "False" ]; then
|
||||
cp -r ./model_repo/audio_tokenizer $model_repo
|
||||
cp -r ./model_repo/speaker_embedding $model_repo
|
||||
fi
|
||||
#fi
|
||||
|
||||
ENGINE_PATH=$trt_engines_dir
|
||||
MAX_QUEUE_DELAY_MICROSECONDS=0
|
||||
MODEL_DIR=$model_scope_model_local_dir
|
||||
LLM_TOKENIZER_DIR=$huggingface_model_local_dir
|
||||
BLS_INSTANCE_NUM=4
|
||||
TRITON_MAX_BATCH_SIZE=16
|
||||
TRITON_MAX_BATCH_SIZE=32
|
||||
DECOUPLED_MODE=True # True for streaming, False for offline
|
||||
STEP_AUDIO_MODEL_DIR=/workspace_yuekai/tts/CosyVoice/runtime/triton_trtllm/Step-Audio-2-mini/token2wav
|
||||
|
||||
python3 scripts/fill_template.py -i ${model_repo}/token2wav/config.pbtxt model_dir:${MODEL_DIR},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS}
|
||||
python3 scripts/fill_template.py -i ${model_repo}/${token2wav_dir}/config.pbtxt model_dir:${STEP_AUDIO_MODEL_DIR},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS}
|
||||
python3 scripts/fill_template.py -i ${model_repo}/${cosyvoice2_dir}/config.pbtxt model_dir:${MODEL_DIR},bls_instance_num:${BLS_INSTANCE_NUM},llm_tokenizer_dir:${LLM_TOKENIZER_DIR},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},decoupled_mode:${DECOUPLED_MODE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS}
|
||||
python3 scripts/fill_template.py -i ${model_repo}/tensorrt_llm/config.pbtxt triton_backend:tensorrtllm,triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},decoupled_mode:${DECOUPLED_MODE},max_beam_width:1,engine_dir:${ENGINE_PATH},max_tokens_in_paged_kv_cache:2560,max_attention_window_size:2560,kv_cache_free_gpu_mem_fraction:0.5,exclude_input_in_output:True,enable_kv_cache_reuse:False,batching_strategy:inflight_fused_batching,max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS},encoder_input_features_data_type:TYPE_FP16,logits_datatype:TYPE_FP32
|
||||
if [ $use_spk2info_cache == "False" ]; then
|
||||
#if [ $use_spk2info_cache == "False" ]; then
|
||||
python3 scripts/fill_template.py -i ${model_repo}/audio_tokenizer/config.pbtxt model_dir:${MODEL_DIR},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS}
|
||||
python3 scripts/fill_template.py -i ${model_repo}/speaker_embedding/config.pbtxt model_dir:${MODEL_DIR},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS}
|
||||
fi
|
||||
#fi
|
||||
rm -rf $model_repo/tensorrt_llm
|
||||
# mv $model_repo/cosyvoice2_dit/1 $model_repo/cosyvoice2_dit/4
|
||||
fi
|
||||
|
||||
if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then
|
||||
echo "Starting Triton server"
|
||||
tritonserver --model-repository $model_repo
|
||||
tritonserver --model-repository $model_repo --http-port 18000
|
||||
fi
|
||||
|
||||
if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then
|
||||
@@ -112,26 +150,26 @@ if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then
|
||||
|
||||
python3 client_grpc.py \
|
||||
--server-addr localhost \
|
||||
--model-name cosyvoice2 \
|
||||
--model-name cosyvoice2_dit \
|
||||
--num-tasks $num_task \
|
||||
--mode $mode \
|
||||
--use-spk2info-cache $use_spk2info_cache \
|
||||
--huggingface-dataset yuekai/seed_tts_cosy2 \
|
||||
--log-dir ./log_concurrent_tasks_${num_task}_${mode}_bls_${BLS_INSTANCE_NUM}_spk_cache_${use_spk2info_cache}
|
||||
--log-dir ./log_concurrent_tasks_${num_task}_${mode}_bls_${BLS_INSTANCE_NUM}_no_att_cnn_cache_new
|
||||
fi
|
||||
|
||||
if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then
|
||||
echo "stage 6: Offline inference benchmark"
|
||||
n_gpus=1
|
||||
datasets=(wenetspeech4tts) # wenetspeech4tts, test_zh, zero_shot_zh
|
||||
backend=trtllm # hf, trtllm, vllm
|
||||
backend=trtllm-serve # hf, trtllm, vllm
|
||||
|
||||
batch_sizes=(16 8 4 2 1)
|
||||
batch_sizes=(16 8 4 2)
|
||||
token2wav_batch_size=1
|
||||
for batch_size in ${batch_sizes[@]}; do
|
||||
for dataset in ${datasets[@]}; do
|
||||
output_dir=./${dataset}_${backend}_llm_batch_size_${batch_size}_token2wav_batch_size_${token2wav_batch_size}
|
||||
CUDA_VISIBLE_DEVICES=0 \
|
||||
CUDA_VISIBLE_DEVICES=1 \
|
||||
python3 offline_inference.py \
|
||||
--output-dir $output_dir \
|
||||
--llm-model-name-or-path $huggingface_model_local_dir \
|
||||
@@ -147,7 +185,31 @@ fi
|
||||
|
||||
if [ $stage -le 7 ] && [ $stop_stage -ge 7 ]; then
|
||||
|
||||
python3 benchmark_streaming_token2wav.py --enable-trt
|
||||
python3 streaming_inference.py
|
||||
|
||||
|
||||
fi
|
||||
|
||||
|
||||
if [ $stage -le 8 ] && [ $stop_stage -ge 8 ]; then
|
||||
mpirun -np 1 --allow-run-as-root --oversubscribe trtllm-serve serve --tokenizer $huggingface_model_local_dir $trt_engines_dir --max_batch_size 16
|
||||
|
||||
fi
|
||||
|
||||
if [ $stage -le 9 ] && [ $stop_stage -ge 9 ]; then
|
||||
#! /usr/bin/env bash
|
||||
curl http://localhost:8000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "trt_engines_bfloat16",
|
||||
"messages":[{"role": "user", "content": "Where is New York?"},
|
||||
{"role": "assistant", "content": "<|s_1708|><|s_2050|><|s_2159|>"}],
|
||||
"max_tokens": 512,
|
||||
"temperature": 0.8,
|
||||
"top_p": 0.95,
|
||||
"top_k": 50,
|
||||
"stop": ["<|eos1|>"],
|
||||
"repetition_penalty": 1.2,
|
||||
"stream": false
|
||||
}'
|
||||
fi
|
||||
Reference in New Issue
Block a user