From 8d67d17f735bad767363bab8fdf8151bba8183a9 Mon Sep 17 00:00:00 2001 From: "lyuxiang.lx" Date: Wed, 16 Apr 2025 20:18:49 +0800 Subject: [PATCH 1/5] update --- README.md | 3 ++- cosyvoice/cli/cosyvoice.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 673def1..4a1dbd3 100644 --- a/README.md +++ b/README.md @@ -134,10 +134,11 @@ prompt_speech_16k = load_wav('./asset/zero_shot_prompt.wav', 16000) for i, j in enumerate(cosyvoice.inference_zero_shot('收到好友从远方寄来的生日礼物，那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐，笑容如花儿般绽放。', '希望你以后能够做的比我还好呦。', prompt_speech_16k, stream=False)): torchaudio.save('zero_shot_{}.wav'.format(i), j['tts_speech'], cosyvoice.sample_rate) -# save zero_shot spk for futher usage +# save zero_shot spk for future usage assert cosyvoice.add_zero_shot_spk('希望你以后能够做的比我还好呦。', prompt_speech_16k, 'my_zero_shot_spk') is True for i, j in enumerate(cosyvoice.inference_zero_shot('收到好友从远方寄来的生日礼物，那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐，笑容如花儿般绽放。', '', '', zero_shot_spk_id='my_zero_shot_spk', stream=False)): torchaudio.save('zero_shot_{}.wav'.format(i), j['tts_speech'], cosyvoice.sample_rate) +cosyvoice.save_spkinfo() # fine grained control, for supported control, check cosyvoice/tokenizer/tokenizer.py#L248 for i, j in enumerate(cosyvoice.inference_cross_lingual('在他讲述那个荒诞故事的过程中，他突然[laughter]停下来，因为他自己也被逗笑了[laughter]。', prompt_speech_16k, stream=False)): diff --git a/cosyvoice/cli/cosyvoice.py b/cosyvoice/cli/cosyvoice.py index efebe4d..a7bfab4 100644 --- a/cosyvoice/cli/cosyvoice.py +++ b/cosyvoice/cli/cosyvoice.py @@ -74,6 +74,9 @@ class CosyVoice: self.frontend.spk2info[zero_shot_spk_id] = model_input return True + def save_spkinfo(self): + torch.save(self.frontend.spk2info, '{}/spk2info.pt'.format(self.model_dir)) + def inference_sft(self, tts_text, spk_id, stream=False, speed=1.0, text_frontend=True): for i in tqdm(self.frontend.text_normalize(tts_text, split=True, text_frontend=text_frontend)): model_input = self.frontend.frontend_sft(i, spk_id) From e97cd1b65563837483e24a7eb24a88e0fe73ddd1 Mon Sep 17 00:00:00 2001 From: "lyuxiang.lx" Date: Sat, 19 Apr 2025 09:08:47 +0800 Subject: [PATCH 2/5] fix cross_lingual bug --- cosyvoice/cli/cosyvoice.py | 4 ++-- cosyvoice/cli/frontend.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cosyvoice/cli/cosyvoice.py b/cosyvoice/cli/cosyvoice.py index a7bfab4..fc1ea90 100644 --- a/cosyvoice/cli/cosyvoice.py +++ b/cosyvoice/cli/cosyvoice.py @@ -102,9 +102,9 @@ class CosyVoice: yield model_output start_time = time.time() - def inference_cross_lingual(self, tts_text, prompt_speech_16k, stream=False, speed=1.0, text_frontend=True): + def inference_cross_lingual(self, tts_text, prompt_speech_16k, zero_shot_spk_id='', stream=False, speed=1.0, text_frontend=True): for i in tqdm(self.frontend.text_normalize(tts_text, split=True, text_frontend=text_frontend)): - model_input = self.frontend.frontend_cross_lingual(i, prompt_speech_16k, self.sample_rate) + model_input = self.frontend.frontend_cross_lingual(i, prompt_speech_16k, self.sample_rate, zero_shot_spk_id) start_time = time.time() logging.info('synthesis text {}'.format(i)) for model_output in self.model.tts(**model_input, stream=stream, speed=speed): diff --git a/cosyvoice/cli/frontend.py b/cosyvoice/cli/frontend.py index 8770e31..99cdb18 100644 --- a/cosyvoice/cli/frontend.py +++ b/cosyvoice/cli/frontend.py @@ -178,8 +178,8 @@ class CosyVoiceFrontEnd: model_input['text_len'] = tts_text_token_len return model_input - def frontend_cross_lingual(self, tts_text, prompt_speech_16k, resample_rate): - model_input = self.frontend_zero_shot(tts_text, '', prompt_speech_16k, resample_rate) + def frontend_cross_lingual(self, tts_text, prompt_speech_16k, resample_rate, zero_shot_spk_id): + model_input = self.frontend_zero_shot(tts_text, '', prompt_speech_16k, resample_rate, zero_shot_spk_id) # in cross lingual mode, we remove prompt in llm del model_input['prompt_text'] del model_input['prompt_text_len'] From 587604b2b433bc350c344b4b181b47249b54faf2 Mon Sep 17 00:00:00 2001 From: bearlu Date: Mon, 21 Apr 2025 09:26:34 -0700 Subject: [PATCH 3/5] fix inference_instruct2 speaker ID bug --- cosyvoice/cli/cosyvoice.py | 4 ++-- cosyvoice/cli/frontend.py | 4 ++-- test1.py | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 test1.py diff --git a/cosyvoice/cli/cosyvoice.py b/cosyvoice/cli/cosyvoice.py index fc1ea90..d82f66e 100644 --- a/cosyvoice/cli/cosyvoice.py +++ b/cosyvoice/cli/cosyvoice.py @@ -177,10 +177,10 @@ class CosyVoice2(CosyVoice): def inference_instruct(self, *args, **kwargs): raise NotImplementedError('inference_instruct is not implemented for CosyVoice2!') - def inference_instruct2(self, tts_text, instruct_text, prompt_speech_16k, stream=False, speed=1.0, text_frontend=True): + def inference_instruct2(self, tts_text, instruct_text, prompt_speech_16k, zero_shot_spk_id='', stream=False, speed=1.0, text_frontend=True): assert isinstance(self.model, CosyVoice2Model), 'inference_instruct2 is only implemented for CosyVoice2!' for i in tqdm(self.frontend.text_normalize(tts_text, split=True, text_frontend=text_frontend)): - model_input = self.frontend.frontend_instruct2(i, instruct_text, prompt_speech_16k, self.sample_rate) + model_input = self.frontend.frontend_instruct2(i, instruct_text, prompt_speech_16k, self.sample_rate, zero_shot_spk_id) start_time = time.time() logging.info('synthesis text {}'.format(i)) for model_output in self.model.tts(**model_input, stream=stream, speed=speed): diff --git a/cosyvoice/cli/frontend.py b/cosyvoice/cli/frontend.py index 99cdb18..36dcd18 100644 --- a/cosyvoice/cli/frontend.py +++ b/cosyvoice/cli/frontend.py @@ -196,8 +196,8 @@ class CosyVoiceFrontEnd: model_input['prompt_text_len'] = instruct_text_token_len return model_input - def frontend_instruct2(self, tts_text, instruct_text, prompt_speech_16k, resample_rate): - model_input = self.frontend_zero_shot(tts_text, instruct_text + '<|endofprompt|>', prompt_speech_16k, resample_rate) + def frontend_instruct2(self, tts_text, instruct_text, prompt_speech_16k, resample_rate, zero_shot_spk_id): + model_input = self.frontend_zero_shot(tts_text, instruct_text + '<|endofprompt|>', prompt_speech_16k, resample_rate, zero_shot_spk_id) del model_input['llm_prompt_speech_token'] del model_input['llm_prompt_speech_token_len'] return model_input diff --git a/test1.py b/test1.py new file mode 100644 index 0000000..a1243e4 --- /dev/null +++ b/test1.py @@ -0,0 +1,37 @@ +import sys +sys.path.append('third_party/Matcha-TTS') +from cosyvoice.cli.cosyvoice import CosyVoice, CosyVoice2 +from cosyvoice.utils.file_utils import load_wav +import torchaudio # type: ignore + +cosyvoice = CosyVoice2('pretrained_models/CosyVoice2-0.5B', load_jit=False, load_trt=False, fp16=False, use_flow_cache=False) + +# NOTE if you want to reproduce the results on https://funaudiollm.github.io/cosyvoice2, please add text_frontend=False during inference +# zero_shot usage +prompt_speech_16k = load_wav('./asset/zero_shot_prompt.wav', 16000) +for i, j in enumerate(cosyvoice.inference_zero_shot('收到好友从远方寄来的生日礼物，那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐，笑容如花儿般绽放。', '希望你以后能够做的比我还好呦。', prompt_speech_16k, stream=False)): + torchaudio.save('zero_shot_{}.wav'.format(i), j['tts_speech'], cosyvoice.sample_rate) + +# save zero_shot spk for future usage +assert cosyvoice.add_zero_shot_spk('希望你以后能够做的比我还好呦。', prompt_speech_16k, 'my_zero_shot_spk') is True +for i, j in enumerate(cosyvoice.inference_zero_shot('收到好友从远方寄来的生日礼物，那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐，笑容如花儿般绽放。', '', '', zero_shot_spk_id='my_zero_shot_spk', stream=False)): + torchaudio.save('zero_shot_{}.wav'.format(i), j['tts_speech'], cosyvoice.sample_rate) +cosyvoice.save_spkinfo() + +# fine grained control, for supported control, check cosyvoice/tokenizer/tokenizer.py#L248 +for i, j in enumerate(cosyvoice.inference_cross_lingual('在他讲述那个荒诞故事的过程中，他突然[laughter]停下来，因为他自己也被逗笑了[laughter]。', prompt_speech_16k, stream=False)): + torchaudio.save('fine_grained_control_{}.wav'.format(i), j['tts_speech'], cosyvoice.sample_rate) + +# instruct usage +for i, j in enumerate(cosyvoice.inference_instruct2('收到好友从远方寄来的生日礼物，那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐，笑容如花儿般绽放。', '用四川话说这句话', prompt_speech_16k, stream=False)): + torchaudio.save('instruct_{}.wav'.format(i), j['tts_speech'], cosyvoice.sample_rate) + +# bistream usage, you can use generator as input, this is useful when using text llm model as input +# NOTE you should still have some basic sentence split logic because llm can not handle arbitrary sentence length +def text_generator(): + yield '收到好友从远方寄来的生日礼物，' + yield '那份意外的惊喜与深深的祝福' + yield '让我心中充满了甜蜜的快乐，' + yield '笑容如花儿般绽放。' +for i, j in enumerate(cosyvoice.inference_zero_shot(text_generator(), '希望你以后能够做的比我还好呦。', prompt_speech_16k, stream=False)): + torchaudio.save('zero_shot_{}.wav'.format(i), j['tts_speech'], cosyvoice.sample_rate) \ No newline at end of file From 65ad448714c60fc6d4133ea3a0439e2ed5320b43 Mon Sep 17 00:00:00 2001 From: burkliu Date: Thu, 24 Apr 2025 17:14:49 +0800 Subject: [PATCH 4/5] [debug] a better solution for mismatch of speech feat len and speech token len, refer to https://github.com/FunAudioLLM/CosyVoice/issues/1051 --- cosyvoice/dataset/processor.py | 12 ++++++++++-- cosyvoice/flow/flow.py | 2 -- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cosyvoice/dataset/processor.py b/cosyvoice/dataset/processor.py index 8424ada..8ac82a1 100644 --- a/cosyvoice/dataset/processor.py +++ b/cosyvoice/dataset/processor.py @@ -159,6 +159,7 @@ def truncate(data, truncate_length=24576, mode='train'): def compute_fbank(data, feat_extractor, + token_mel_ratio=2, mode='train'): """ Extract fbank @@ -174,8 +175,15 @@ def compute_fbank(data, assert 'utt' in sample assert 'text_token' in sample waveform = sample['speech'] - mat = feat_extractor(waveform).squeeze(dim=0).transpose(0, 1) - sample['speech_feat'] = mat + feat = feat_extractor(waveform).squeeze(dim=0).transpose(0, 1) + + # padding with replicate mode (align to speech_token len * token_mel_ratio) + pad_len = sample["speech_token"].shape[0] * token_mel_ratio - feat.shape[0] + if pad_len > 0: + feat_to_pad = feat[-1:].repeat((pad_len, 1)) + feat = torch.cat([feat, feat_to_pad], dim=0) + + sample['speech_feat'] = feat yield sample diff --git a/cosyvoice/flow/flow.py b/cosyvoice/flow/flow.py index 9c642ee..e1cf429 100644 --- a/cosyvoice/flow/flow.py +++ b/cosyvoice/flow/flow.py @@ -92,7 +92,6 @@ class MaskedDiffWithXvec(torch.nn.Module): mask = (~make_pad_mask(feat_len)).to(h) # NOTE this is unnecessary, feat/h already same shape - feat = F.interpolate(feat.unsqueeze(dim=1), size=h.shape[1:], mode="nearest").squeeze(dim=1) loss, _ = self.decoder.compute_loss( feat.transpose(1, 2).contiguous(), mask.unsqueeze(1), @@ -214,7 +213,6 @@ class CausalMaskedDiffWithXvec(torch.nn.Module): h = self.encoder_proj(h) # get conditions - feat = F.interpolate(feat.unsqueeze(dim=1), size=h.shape[1:], mode="nearest").squeeze(dim=1) conds = torch.zeros(feat.shape, device=token.device) for i, j in enumerate(feat_len): if random.random() < 0.5: From 038ff9f353b21c98c54b744eaa19ba9b3674c35a Mon Sep 17 00:00:00 2001 From: burkliu Date: Fri, 25 Apr 2025 10:31:43 +0800 Subject: [PATCH 5/5] [feature] modify pad to trim --- cosyvoice/dataset/processor.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cosyvoice/dataset/processor.py b/cosyvoice/dataset/processor.py index 8ac82a1..08030d6 100644 --- a/cosyvoice/dataset/processor.py +++ b/cosyvoice/dataset/processor.py @@ -177,11 +177,10 @@ def compute_fbank(data, waveform = sample['speech'] feat = feat_extractor(waveform).squeeze(dim=0).transpose(0, 1) - # padding with replicate mode (align to speech_token len * token_mel_ratio) - pad_len = sample["speech_token"].shape[0] * token_mel_ratio - feat.shape[0] - if pad_len > 0: - feat_to_pad = feat[-1:].repeat((pad_len, 1)) - feat = torch.cat([feat, feat_to_pad], dim=0) + # trim to align speech_token and speech_feat + token_len = min(feat.shape[0] // token_mel_ratio, sample["speech_token"].shape[0]) + feat = feat[:token_mel_ratio * token_len] + sample["speech_token"] = sample["speech_token"][:token_len] sample['speech_feat'] = feat yield sample