mirror of
https://github.com/FunAudioLLM/CosyVoice.git
synced 2026-02-05 18:09:24 +08:00
267 lines
13 KiB
Bash
267 lines
13 KiB
Bash
#!/bin/bash
|
||
# Copyright (c) 2025 NVIDIA (authors: Yuekai Zhang)
|
||
export CUDA_VISIBLE_DEVICES=0
|
||
cosyvoice_path=/workspace/CosyVoice
|
||
cosyvoice_path=/workspace_yuekai/tts/CosyVoice
|
||
stepaudio2_path=/workspace_yuekai/tts/Step-Audio2
|
||
export PYTHONPATH=${stepaudio2_path}:$PYTHONPATH
|
||
export PYTHONPATH=${cosyvoice_path}:$PYTHONPATH
|
||
export PYTHONPATH=${cosyvoice_path}/third_party/Matcha-TTS:$PYTHONPATH
|
||
stage=$1
|
||
stop_stage=$2
|
||
N_GPUS=2 # set the number of GPUs to use
|
||
|
||
|
||
huggingface_model_local_dir=./cosyvoice2_llm
|
||
model_scope_model_local_dir=./CosyVoice2-0.5B
|
||
trt_dtype=bfloat16
|
||
trt_weights_dir=./trt_weights_${trt_dtype}
|
||
trt_engines_dir=./trt_engines_${trt_dtype}
|
||
|
||
model_repo=./model_repo_cosyvoice2_dit
|
||
|
||
use_spk2info_cache=False
|
||
|
||
if [ $stage -le -1 ] && [ $stop_stage -ge -1 ]; then
|
||
echo "Cloning CosyVoice"
|
||
git clone --recursive https://github.com/FunAudioLLM/CosyVoice.git $cosyvoice_path
|
||
cd $cosyvoice_path
|
||
git submodule update --init --recursive
|
||
cd runtime/triton_trtllm
|
||
fi
|
||
|
||
if [ $stage -le 0 ] && [ $stop_stage -ge 0 ]; then
|
||
echo "Downloading CosyVoice2-0.5B"
|
||
# see https://github.com/nvidia-china-sae/mair-hub/blob/main/rl-tutorial/cosyvoice_llm/pretrained_to_huggingface.py
|
||
huggingface-cli download --local-dir $huggingface_model_local_dir yuekai/cosyvoice2_llm
|
||
modelscope download --model iic/CosyVoice2-0.5B --local_dir $model_scope_model_local_dir
|
||
# download spk2info.pt to directly use cached speech tokens, speech feats, and embeddings
|
||
wget https://raw.githubusercontent.com/qi-hua/async_cosyvoice/main/CosyVoice2-0.5B/spk2info.pt -O $model_scope_model_local_dir/spk2info.pt
|
||
fi
|
||
|
||
|
||
if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then
|
||
echo "Converting checkpoint to TensorRT weights"
|
||
python3 scripts/convert_checkpoint.py --model_dir $huggingface_model_local_dir \
|
||
--output_dir $trt_weights_dir \
|
||
--dtype $trt_dtype || exit 1
|
||
|
||
echo "Building TensorRT engines"
|
||
trtllm-build --checkpoint_dir $trt_weights_dir \
|
||
--output_dir $trt_engines_dir \
|
||
--max_batch_size 16 \
|
||
--max_num_tokens 32768 \
|
||
--gemm_plugin $trt_dtype || exit 1
|
||
|
||
echo "Testing TensorRT engines"
|
||
python3 ./scripts/test_llm.py --input_text "你好，请问你叫什么？" \
|
||
--tokenizer_dir $huggingface_model_local_dir \
|
||
--top_k 50 --top_p 0.95 --temperature 0.8 \
|
||
--engine_dir=$trt_engines_dir || exit 1
|
||
fi
|
||
|
||
|
||
# if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then
|
||
# echo "Creating model repository"
|
||
# rm -rf $model_repo
|
||
# mkdir -p $model_repo
|
||
# cosyvoice2_dir="cosyvoice2_dit"
|
||
# token2wav_dir="token2wav_dit"
|
||
|
||
# cp -r ./model_repo/${cosyvoice2_dir} $model_repo
|
||
# cp -r ./model_repo/tensorrt_llm $model_repo
|
||
# cp -r ./model_repo/${token2wav_dir} $model_repo
|
||
# #if [ $use_spk2info_cache == "False" ]; then
|
||
# cp -r ./model_repo/audio_tokenizer $model_repo
|
||
# cp -r ./model_repo/speaker_embedding $model_repo
|
||
# #fi
|
||
|
||
# ENGINE_PATH=$trt_engines_dir
|
||
# MAX_QUEUE_DELAY_MICROSECONDS=0
|
||
# MODEL_DIR=$model_scope_model_local_dir
|
||
# LLM_TOKENIZER_DIR=$huggingface_model_local_dir
|
||
# BLS_INSTANCE_NUM=1
|
||
# TRITON_MAX_BATCH_SIZE=16
|
||
# DECOUPLED_MODE=True # True for streaming, False for offline
|
||
# STEP_AUDIO_MODEL_DIR=/workspace_yuekai/tts/CosyVoice/runtime/triton_trtllm/Step-Audio-2-mini/token2wav
|
||
|
||
# python3 scripts/fill_template.py -i ${model_repo}/${token2wav_dir}/config.pbtxt model_dir:${STEP_AUDIO_MODEL_DIR},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS}
|
||
# python3 scripts/fill_template.py -i ${model_repo}/${cosyvoice2_dir}/config.pbtxt model_dir:${MODEL_DIR},bls_instance_num:${BLS_INSTANCE_NUM},llm_tokenizer_dir:${LLM_TOKENIZER_DIR},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},decoupled_mode:${DECOUPLED_MODE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS}
|
||
# python3 scripts/fill_template.py -i ${model_repo}/tensorrt_llm/config.pbtxt triton_backend:tensorrtllm,triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},decoupled_mode:${DECOUPLED_MODE},max_beam_width:1,engine_dir:${ENGINE_PATH},max_tokens_in_paged_kv_cache:2560,max_attention_window_size:2560,kv_cache_free_gpu_mem_fraction:0.5,exclude_input_in_output:True,enable_kv_cache_reuse:False,batching_strategy:inflight_fused_batching,max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS},encoder_input_features_data_type:TYPE_FP16,logits_datatype:TYPE_FP32
|
||
# #if [ $use_spk2info_cache == "False" ]; then
|
||
# python3 scripts/fill_template.py -i ${model_repo}/audio_tokenizer/config.pbtxt model_dir:${MODEL_DIR},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS}
|
||
# python3 scripts/fill_template.py -i ${model_repo}/speaker_embedding/config.pbtxt model_dir:${MODEL_DIR},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS}
|
||
# #fi
|
||
# fi
|
||
|
||
if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then
|
||
echo "Creating model repository async mode"
|
||
rm -rf $model_repo
|
||
mkdir -p $model_repo
|
||
cosyvoice2_dir="cosyvoice2_dit"
|
||
token2wav_dir="token2wav_dit"
|
||
|
||
cp -r ./model_repo/${cosyvoice2_dir} $model_repo
|
||
cp -r ./model_repo/tensorrt_llm $model_repo
|
||
cp -r ./model_repo/${token2wav_dir} $model_repo
|
||
#if [ $use_spk2info_cache == "False" ]; then
|
||
cp -r ./model_repo/audio_tokenizer $model_repo
|
||
cp -r ./model_repo/speaker_embedding $model_repo
|
||
#fi
|
||
|
||
ENGINE_PATH=$trt_engines_dir
|
||
MAX_QUEUE_DELAY_MICROSECONDS=0
|
||
MODEL_DIR=$model_scope_model_local_dir
|
||
LLM_TOKENIZER_DIR=$huggingface_model_local_dir
|
||
BLS_INSTANCE_NUM=4
|
||
TRITON_MAX_BATCH_SIZE=1
|
||
DECOUPLED_MODE=True # True for streaming, False for offline
|
||
STEP_AUDIO_MODEL_DIR=/workspace_yuekai/tts/CosyVoice/runtime/triton_trtllm/Step-Audio-2-mini/token2wav
|
||
|
||
python3 scripts/fill_template.py -i ${model_repo}/${token2wav_dir}/config.pbtxt model_dir:${STEP_AUDIO_MODEL_DIR},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS}
|
||
python3 scripts/fill_template.py -i ${model_repo}/${cosyvoice2_dir}/config.pbtxt model_dir:${MODEL_DIR},bls_instance_num:${BLS_INSTANCE_NUM},llm_tokenizer_dir:${LLM_TOKENIZER_DIR},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},decoupled_mode:${DECOUPLED_MODE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS}
|
||
python3 scripts/fill_template.py -i ${model_repo}/tensorrt_llm/config.pbtxt triton_backend:tensorrtllm,triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},decoupled_mode:${DECOUPLED_MODE},max_beam_width:1,engine_dir:${ENGINE_PATH},max_tokens_in_paged_kv_cache:2560,max_attention_window_size:2560,kv_cache_free_gpu_mem_fraction:0.5,exclude_input_in_output:True,enable_kv_cache_reuse:False,batching_strategy:inflight_fused_batching,max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS},encoder_input_features_data_type:TYPE_FP16,logits_datatype:TYPE_FP32
|
||
#if [ $use_spk2info_cache == "False" ]; then
|
||
python3 scripts/fill_template.py -i ${model_repo}/audio_tokenizer/config.pbtxt model_dir:${MODEL_DIR},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS}
|
||
python3 scripts/fill_template.py -i ${model_repo}/speaker_embedding/config.pbtxt model_dir:${MODEL_DIR},triton_max_batch_size:${TRITON_MAX_BATCH_SIZE},max_queue_delay_microseconds:${MAX_QUEUE_DELAY_MICROSECONDS}
|
||
#fi
|
||
rm -rf $model_repo/tensorrt_llm
|
||
# mv $model_repo/cosyvoice2_dit/1 $model_repo/cosyvoice2_dit/4
|
||
fi
|
||
|
||
if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then
|
||
echo "Starting Triton server on $N_GPUS GPUs"
|
||
for i in $(seq 0 $(($N_GPUS - 1))); do
|
||
echo "Starting server on GPU $i"
|
||
http_port=$((19000 + $i))
|
||
grpc_port=$((18000 + $i))
|
||
metrics_port=$((17000 + $i))
|
||
CUDA_VISIBLE_DEVICES=$i tritonserver --model-repository $model_repo --http-port $http_port --grpc-port $grpc_port --metrics-port $metrics_port &
|
||
done
|
||
|
||
echo "Servers are running in the background. Press Ctrl+C to stop them and the script."
|
||
wait
|
||
fi
|
||
|
||
if [ $stage -le 30 ] && [ $stop_stage -ge 30 ]; then
|
||
echo "Starting Triton server on $N_GPUS GPUs"
|
||
N_GPUS=1
|
||
for i in $(seq 0 $(($N_GPUS - 1))); do
|
||
echo "Starting server on GPU $i"
|
||
http_port=$((19000 + $i))
|
||
grpc_port=$((18000 + $i))
|
||
metrics_port=$((17000 + $i))
|
||
CUDA_VISIBLE_DEVICES=0 tritonserver --model-repository $model_repo --http-port $http_port --grpc-port $grpc_port --metrics-port $metrics_port &
|
||
done
|
||
|
||
echo "Servers are running in the background. Press Ctrl+C to stop them and the script."
|
||
wait
|
||
fi
|
||
|
||
if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then
|
||
echo "Single request test http, only work for offline TTS mode"
|
||
python3 client_http.py \
|
||
--reference-audio ./assets/prompt_audio.wav \
|
||
--reference-text "吃燕窝就选燕之屋，本节目由26年专注高品质燕窝的燕之屋冠名播出。豆奶牛奶换着喝，营养更均衡，本节目由豆本豆豆奶特约播出。" \
|
||
--target-text "身临其境，换新体验。塑造开源语音合成新范式，让智能语音更自然。" \
|
||
--model-name cosyvoice2
|
||
fi
|
||
|
||
if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then
|
||
echo "Running benchmark client grpc on $N_GPUS GPUs"
|
||
num_task=1
|
||
|
||
mode=streaming
|
||
BLS_INSTANCE_NUM=4
|
||
|
||
for i in $(seq 0 $(($N_GPUS - 1))); do
|
||
grpc_port=$((18000 + $i))
|
||
echo "Running client for server on localhost:$grpc_port"
|
||
python3 client_grpc.py \
|
||
--server-addr localhost \
|
||
--server-port $grpc_port \
|
||
--model-name cosyvoice2_dit \
|
||
--num-tasks $num_task \
|
||
--mode $mode \
|
||
--huggingface-dataset yuekai/seed_tts_cosy2 \
|
||
--log-dir ./log_debug_concurrent_tasks_${num_task}_${mode}_bls_${BLS_INSTANCE_NUM}_gpu${i} &
|
||
done
|
||
wait
|
||
fi
|
||
if [ $stage -le 50 ] && [ $stop_stage -ge 50 ]; then
|
||
echo "Running benchmark client grpc on $N_GPUS GPUs"
|
||
num_task=4
|
||
N_GPUS=1
|
||
mode=streaming
|
||
BLS_INSTANCE_NUM=4
|
||
|
||
for i in $(seq 0 $(($N_GPUS - 1))); do
|
||
grpc_port=$((18000 + $i))
|
||
echo "Running client for server on localhost:$grpc_port"
|
||
python3 client_grpc.py \
|
||
--server-addr localhost \
|
||
--server-port $grpc_port \
|
||
--model-name cosyvoice2_dit \
|
||
--num-tasks $num_task \
|
||
--mode $mode \
|
||
--huggingface-dataset yuekai/seed_tts_cosy2 \
|
||
--log-dir ./log_single_card_concurrent_tasks_${num_task}_${mode}_bls_${BLS_INSTANCE_NUM} &
|
||
done
|
||
wait
|
||
fi
|
||
if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then
|
||
echo "stage 6: Offline inference benchmark"
|
||
n_gpus=1
|
||
datasets=(wenetspeech4tts) # wenetspeech4tts, test_zh, zero_shot_zh
|
||
backend=trtllm-serve # hf, trtllm, vllm
|
||
|
||
batch_sizes=(16 8 4 2 1)
|
||
batch_sizes=(16 8 4 2)
|
||
token2wav_batch_size=1
|
||
for batch_size in ${batch_sizes[@]}; do
|
||
for dataset in ${datasets[@]}; do
|
||
output_dir=./${dataset}_${backend}_llm_batch_size_${batch_size}_token2wav_batch_size_${token2wav_batch_size}
|
||
CUDA_VISIBLE_DEVICES=1 \
|
||
python3 offline_inference.py \
|
||
--output-dir $output_dir \
|
||
--llm-model-name-or-path $huggingface_model_local_dir \
|
||
--token2wav-path $model_scope_model_local_dir \
|
||
--backend $backend \
|
||
--batch-size $batch_size --token2wav-batch-size $token2wav_batch_size \
|
||
--engine-dir $trt_engines_dir \
|
||
--split-name ${dataset} || exit 1
|
||
done
|
||
done
|
||
fi
|
||
|
||
|
||
if [ $stage -le 7 ] && [ $stop_stage -ge 7 ]; then
|
||
|
||
CUDA_VISIBLE_DEVICES=2 python3 streaming_inference.py --enable-trt --strategy exponential
|
||
|
||
|
||
fi
|
||
|
||
|
||
if [ $stage -le 8 ] && [ $stop_stage -ge 8 ]; then
|
||
CUDA_VISIBLE_DEVICES=0 mpirun -np 1 --allow-run-as-root --oversubscribe trtllm-serve serve --tokenizer $huggingface_model_local_dir $trt_engines_dir --max_batch_size 16 --kv_cache_free_gpu_memory_fraction 0.4
|
||
|
||
fi
|
||
|
||
if [ $stage -le 9 ] && [ $stop_stage -ge 9 ]; then
|
||
#! /usr/bin/env bash
|
||
curl http://localhost:8000/v1/chat/completions \
|
||
-H "Content-Type: application/json" \
|
||
-d '{
|
||
"model": "trt_engines_bfloat16",
|
||
"messages":[{"role": "user", "content": "Where is New York?"},
|
||
{"role": "assistant", "content": "<|s_1708|><|s_2050|><|s_2159|>"}],
|
||
"max_tokens": 512,
|
||
"temperature": 0.8,
|
||
"top_p": 0.95,
|
||
"top_k": 50,
|
||
"stop": ["<|eos1|>"],
|
||
"repetition_penalty": 1.2,
|
||
"stream": false
|
||
}'
|
||
fi |