deepgeek/CosyVoice
2
0
Fork 0
mirror of https://github.com/FunAudioLLM/CosyVoice.git synced 2026-02-05 18:09:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

add text_frontend arg

Browse Source
This commit is contained in:
lyuxiang.lx
2024-12-17 14:03:35 +08:00
parent 60b0416229
commit 26c774098d
3 changed files with 16 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@@ -137,6 +137,7 @@ import torchaudio
```python ```python
cosyvoice = CosyVoice2('pretrained_models/CosyVoice2-0.5B', load_jit=True, load_onnx=False, load_trt=False) cosyvoice = CosyVoice2('pretrained_models/CosyVoice2-0.5B', load_jit=True, load_onnx=False, load_trt=False)
# NOTE if you want to reproduce the results on https://funaudiollm.github.io/cosyvoice2, please add text_frontend=False during inference
# zero_shot usage # zero_shot usage
prompt_speech_16k = load_wav('zero_shot_prompt.wav', 16000) prompt_speech_16k = load_wav('zero_shot_prompt.wav', 16000)
for i, j in enumerate(cosyvoice.inference_zero_shot('收到好友从远方寄来的生日礼物，那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐，笑容如花儿般绽放。', '希望你以后能够做的比我还好呦。', prompt_speech_16k, stream=False)): for i, j in enumerate(cosyvoice.inference_zero_shot('收到好友从远方寄来的生日礼物，那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐，笑容如花儿般绽放。', '希望你以后能够做的比我还好呦。', prompt_speech_16k, stream=False)):

24
cosyvoice/cli/cosyvoice.py
View File

@@ -59,8 +59,8 @@ class CosyVoice:
spks = list(self.frontend.spk2info.keys()) spks = list(self.frontend.spk2info.keys())
return spks return spks
def inference_sft(self, tts_text, spk_id, stream=False, speed=1.0): def inference_sft(self, tts_text, spk_id, stream=False, speed=1.0, text_frontend=True):
for i in tqdm(self.frontend.text_normalize(tts_text, split=True)): for i in tqdm(self.frontend.text_normalize(tts_text, split=True, text_frontend=text_frontend)):
model_input = self.frontend.frontend_sft(i, spk_id) model_input = self.frontend.frontend_sft(i, spk_id)
start_time = time.time() start_time = time.time()
logging.info('synthesis text {}'.format(i)) logging.info('synthesis text {}'.format(i))
@@ -70,9 +70,9 @@ class CosyVoice:
yield model_output yield model_output
start_time = time.time() start_time = time.time()
def inference_zero_shot(self, tts_text, prompt_text, prompt_speech_16k, stream=False, speed=1.0): def inference_zero_shot(self, tts_text, prompt_text, prompt_speech_16k, stream=False, speed=1.0, text_frontend=True):
prompt_text = self.frontend.text_normalize(prompt_text, split=False) prompt_text = self.frontend.text_normalize(prompt_text, split=False, text_frontend=text_frontend)
for i in tqdm(self.frontend.text_normalize(tts_text, split=True)): for i in tqdm(self.frontend.text_normalize(tts_text, split=True, text_frontend=text_frontend)):
if len(i) < 0.5 * len(prompt_text): if len(i) < 0.5 * len(prompt_text):
logging.warning('synthesis text {} too short than prompt text {}, this may lead to bad performance'.format(i, prompt_text)) logging.warning('synthesis text {} too short than prompt text {}, this may lead to bad performance'.format(i, prompt_text))
model_input = self.frontend.frontend_zero_shot(i, prompt_text, prompt_speech_16k, self.sample_rate) model_input = self.frontend.frontend_zero_shot(i, prompt_text, prompt_speech_16k, self.sample_rate)
@@ -84,10 +84,10 @@ class CosyVoice:
yield model_output yield model_output
start_time = time.time() start_time = time.time()
def inference_cross_lingual(self, tts_text, prompt_speech_16k, stream=False, speed=1.0): def inference_cross_lingual(self, tts_text, prompt_speech_16k, stream=False, speed=1.0, text_frontend=True):
if self.frontend.instruct is True and isinstance(self.model, CosyVoiceModel): if self.frontend.instruct is True and isinstance(self.model, CosyVoiceModel):
raise ValueError('{} do not support cross_lingual inference'.format(self.model_dir)) raise ValueError('{} do not support cross_lingual inference'.format(self.model_dir))
for i in tqdm(self.frontend.text_normalize(tts_text, split=True)): for i in tqdm(self.frontend.text_normalize(tts_text, split=True, text_frontend=text_frontend)):
model_input = self.frontend.frontend_cross_lingual(i, prompt_speech_16k, self.sample_rate) model_input = self.frontend.frontend_cross_lingual(i, prompt_speech_16k, self.sample_rate)
start_time = time.time() start_time = time.time()
logging.info('synthesis text {}'.format(i)) logging.info('synthesis text {}'.format(i))
@@ -97,12 +97,12 @@ class CosyVoice:
yield model_output yield model_output
start_time = time.time() start_time = time.time()
def inference_instruct(self, tts_text, spk_id, instruct_text, stream=False, speed=1.0): def inference_instruct(self, tts_text, spk_id, instruct_text, stream=False, speed=1.0, text_frontend=True):
assert isinstance(self.model, CosyVoiceModel) assert isinstance(self.model, CosyVoiceModel)
if self.frontend.instruct is False: if self.frontend.instruct is False:
raise ValueError('{} do not support instruct inference'.format(self.model_dir)) raise ValueError('{} do not support instruct inference'.format(self.model_dir))
instruct_text = self.frontend.text_normalize(instruct_text, split=False) instruct_text = self.frontend.text_normalize(instruct_text, split=False, text_frontend=text_frontend)
for i in tqdm(self.frontend.text_normalize(tts_text, split=True)): for i in tqdm(self.frontend.text_normalize(tts_text, split=True, text_frontend=text_frontend)):
model_input = self.frontend.frontend_instruct(i, spk_id, instruct_text) model_input = self.frontend.frontend_instruct(i, spk_id, instruct_text)
start_time = time.time() start_time = time.time()
logging.info('synthesis text {}'.format(i)) logging.info('synthesis text {}'.format(i))
@@ -112,9 +112,9 @@ class CosyVoice:
yield model_output yield model_output
start_time = time.time() start_time = time.time()
def inference_instruct2(self, tts_text, instruct_text, prompt_speech_16k, stream=False, speed=1.0): def inference_instruct2(self, tts_text, instruct_text, prompt_speech_16k, stream=False, speed=1.0, text_frontend=True):
assert isinstance(self.model, CosyVoice2Model) assert isinstance(self.model, CosyVoice2Model)
for i in tqdm(self.frontend.text_normalize(tts_text, split=True)): for i in tqdm(self.frontend.text_normalize(tts_text, split=True, text_frontend=text_frontend)):
model_input = self.frontend.frontend_instruct2(i, instruct_text, prompt_speech_16k, self.sample_rate) model_input = self.frontend.frontend_instruct2(i, instruct_text, prompt_speech_16k, self.sample_rate)
start_time = time.time() start_time = time.time()
logging.info('synthesis text {}'.format(i)) logging.info('synthesis text {}'.format(i))

8
cosyvoice/cli/frontend.py
View File

@@ -107,12 +107,10 @@ class CosyVoiceFrontEnd:
speech_feat_len = torch.tensor([speech_feat.shape[1]], dtype=torch.int32).to(self.device) speech_feat_len = torch.tensor([speech_feat.shape[1]], dtype=torch.int32).to(self.device)
return speech_feat, speech_feat_len return speech_feat, speech_feat_len
def text_normalize(self, text, split=True): def text_normalize(self, text, split=True, text_frontend=True):
if text_frontend is False:
return [text] if split is True else text
text = text.strip() text = text.strip()
# NOTE(lyuxiang.lx) move this judgement into ttsfrd in the future
for token in self.tokenizer.special_tokens['additional_special_tokens']:
if token in text:
return text if split is False else [text]
if contains_chinese(text): if contains_chinese(text):
if self.use_ttsfrd: if self.use_ttsfrd:
texts = [i["text"] for i in json.loads(self.frd.do_voicegen_frd(text))["sentences"]] texts = [i["text"] for i in json.loads(self.frd.do_voicegen_frd(text))["sentences"]]