feat: CosyVoice 한국어 엔진 통합 (Apache-2.0, GPU)
- vendor/CosyVoice: 패치된 CosyVoice 코드 + Matcha-TTS 벤더링 - cosyvoice_worker: 전용 venv에서 상주하는 합성 워커(FastAPI, 모델 1회 로드) - app/engines/cosyvoice_engine: HTTP 프록시 엔진(속도/피치) - Dockerfile: 전용 venv(torch cu128) + 모델 베이킹 + 빌드 워밍업 - entrypoint.sh: 워커(8001) + 메인앱(8788) 동시 기동
This commit is contained in:
claude
1
vendor/CosyVoice/third_party/Matcha-TTS/matcha/VERSION
vendored
Normal file
1
vendor/CosyVoice/third_party/Matcha-TTS/matcha/VERSION
vendored
Normal file
@@ -0,0 +1 @@
|
||||
0.0.5.1
|
||||
0
vendor/CosyVoice/third_party/Matcha-TTS/matcha/__init__.py
vendored
Normal file
0
vendor/CosyVoice/third_party/Matcha-TTS/matcha/__init__.py
vendored
Normal file
357
vendor/CosyVoice/third_party/Matcha-TTS/matcha/app.py
vendored
Normal file
357
vendor/CosyVoice/third_party/Matcha-TTS/matcha/app.py
vendored
Normal file
@@ -0,0 +1,357 @@
|
||||
import tempfile
|
||||
from argparse import Namespace
|
||||
from pathlib import Path
|
||||
|
||||
import gradio as gr
|
||||
import soundfile as sf
|
||||
import torch
|
||||
|
||||
from matcha.cli import (
|
||||
MATCHA_URLS,
|
||||
VOCODER_URLS,
|
||||
assert_model_downloaded,
|
||||
get_device,
|
||||
load_matcha,
|
||||
load_vocoder,
|
||||
process_text,
|
||||
to_waveform,
|
||||
)
|
||||
from matcha.utils.utils import get_user_data_dir, plot_tensor
|
||||
|
||||
LOCATION = Path(get_user_data_dir())
|
||||
|
||||
args = Namespace(
|
||||
cpu=False,
|
||||
model="matcha_vctk",
|
||||
vocoder="hifigan_univ_v1",
|
||||
spk=0,
|
||||
)
|
||||
|
||||
CURRENTLY_LOADED_MODEL = args.model
|
||||
|
||||
|
||||
def MATCHA_TTS_LOC(x):
|
||||
return LOCATION / f"{x}.ckpt"
|
||||
|
||||
|
||||
def VOCODER_LOC(x):
|
||||
return LOCATION / f"{x}"
|
||||
|
||||
|
||||
LOGO_URL = "https://shivammehta25.github.io/Matcha-TTS/images/logo.png"
|
||||
RADIO_OPTIONS = {
|
||||
"Multi Speaker (VCTK)": {
|
||||
"model": "matcha_vctk",
|
||||
"vocoder": "hifigan_univ_v1",
|
||||
},
|
||||
"Single Speaker (LJ Speech)": {
|
||||
"model": "matcha_ljspeech",
|
||||
"vocoder": "hifigan_T2_v1",
|
||||
},
|
||||
}
|
||||
|
||||
# Ensure all the required models are downloaded
|
||||
assert_model_downloaded(MATCHA_TTS_LOC("matcha_ljspeech"), MATCHA_URLS["matcha_ljspeech"])
|
||||
assert_model_downloaded(VOCODER_LOC("hifigan_T2_v1"), VOCODER_URLS["hifigan_T2_v1"])
|
||||
assert_model_downloaded(MATCHA_TTS_LOC("matcha_vctk"), MATCHA_URLS["matcha_vctk"])
|
||||
assert_model_downloaded(VOCODER_LOC("hifigan_univ_v1"), VOCODER_URLS["hifigan_univ_v1"])
|
||||
|
||||
device = get_device(args)
|
||||
|
||||
# Load default model
|
||||
model = load_matcha(args.model, MATCHA_TTS_LOC(args.model), device)
|
||||
vocoder, denoiser = load_vocoder(args.vocoder, VOCODER_LOC(args.vocoder), device)
|
||||
|
||||
|
||||
def load_model(model_name, vocoder_name):
|
||||
model = load_matcha(model_name, MATCHA_TTS_LOC(model_name), device)
|
||||
vocoder, denoiser = load_vocoder(vocoder_name, VOCODER_LOC(vocoder_name), device)
|
||||
return model, vocoder, denoiser
|
||||
|
||||
|
||||
def load_model_ui(model_type, textbox):
|
||||
model_name, vocoder_name = RADIO_OPTIONS[model_type]["model"], RADIO_OPTIONS[model_type]["vocoder"]
|
||||
|
||||
global model, vocoder, denoiser, CURRENTLY_LOADED_MODEL # pylint: disable=global-statement
|
||||
if CURRENTLY_LOADED_MODEL != model_name:
|
||||
model, vocoder, denoiser = load_model(model_name, vocoder_name)
|
||||
CURRENTLY_LOADED_MODEL = model_name
|
||||
|
||||
if model_name == "matcha_ljspeech":
|
||||
spk_slider = gr.update(visible=False, value=-1)
|
||||
single_speaker_examples = gr.update(visible=True)
|
||||
multi_speaker_examples = gr.update(visible=False)
|
||||
length_scale = gr.update(value=0.95)
|
||||
else:
|
||||
spk_slider = gr.update(visible=True, value=0)
|
||||
single_speaker_examples = gr.update(visible=False)
|
||||
multi_speaker_examples = gr.update(visible=True)
|
||||
length_scale = gr.update(value=0.85)
|
||||
|
||||
return (
|
||||
textbox,
|
||||
gr.update(interactive=True),
|
||||
spk_slider,
|
||||
single_speaker_examples,
|
||||
multi_speaker_examples,
|
||||
length_scale,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def process_text_gradio(text):
|
||||
output = process_text(1, text, device)
|
||||
return output["x_phones"][1::2], output["x"], output["x_lengths"]
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def synthesise_mel(text, text_length, n_timesteps, temperature, length_scale, spk):
|
||||
spk = torch.tensor([spk], device=device, dtype=torch.long) if spk >= 0 else None
|
||||
output = model.synthesise(
|
||||
text,
|
||||
text_length,
|
||||
n_timesteps=n_timesteps,
|
||||
temperature=temperature,
|
||||
spks=spk,
|
||||
length_scale=length_scale,
|
||||
)
|
||||
output["waveform"] = to_waveform(output["mel"], vocoder, denoiser)
|
||||
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as fp:
|
||||
sf.write(fp.name, output["waveform"], 22050, "PCM_24")
|
||||
|
||||
return fp.name, plot_tensor(output["mel"].squeeze().cpu().numpy())
|
||||
|
||||
|
||||
def multispeaker_example_cacher(text, n_timesteps, mel_temp, length_scale, spk):
|
||||
global CURRENTLY_LOADED_MODEL # pylint: disable=global-statement
|
||||
if CURRENTLY_LOADED_MODEL != "matcha_vctk":
|
||||
global model, vocoder, denoiser # pylint: disable=global-statement
|
||||
model, vocoder, denoiser = load_model("matcha_vctk", "hifigan_univ_v1")
|
||||
CURRENTLY_LOADED_MODEL = "matcha_vctk"
|
||||
|
||||
phones, text, text_lengths = process_text_gradio(text)
|
||||
audio, mel_spectrogram = synthesise_mel(text, text_lengths, n_timesteps, mel_temp, length_scale, spk)
|
||||
return phones, audio, mel_spectrogram
|
||||
|
||||
|
||||
def ljspeech_example_cacher(text, n_timesteps, mel_temp, length_scale, spk=-1):
|
||||
global CURRENTLY_LOADED_MODEL # pylint: disable=global-statement
|
||||
if CURRENTLY_LOADED_MODEL != "matcha_ljspeech":
|
||||
global model, vocoder, denoiser # pylint: disable=global-statement
|
||||
model, vocoder, denoiser = load_model("matcha_ljspeech", "hifigan_T2_v1")
|
||||
CURRENTLY_LOADED_MODEL = "matcha_ljspeech"
|
||||
|
||||
phones, text, text_lengths = process_text_gradio(text)
|
||||
audio, mel_spectrogram = synthesise_mel(text, text_lengths, n_timesteps, mel_temp, length_scale, spk)
|
||||
return phones, audio, mel_spectrogram
|
||||
|
||||
|
||||
def main():
|
||||
description = """# 🍵 Matcha-TTS: A fast TTS architecture with conditional flow matching
|
||||
### [Shivam Mehta](https://www.kth.se/profile/smehta), [Ruibo Tu](https://www.kth.se/profile/ruibo), [Jonas Beskow](https://www.kth.se/profile/beskow), [Éva Székely](https://www.kth.se/profile/szekely), and [Gustav Eje Henter](https://people.kth.se/~ghe/)
|
||||
We propose 🍵 Matcha-TTS, a new approach to non-autoregressive neural TTS, that uses conditional flow matching (similar to rectified flows) to speed up ODE-based speech synthesis. Our method:
|
||||
|
||||
|
||||
* Is probabilistic
|
||||
* Has compact memory footprint
|
||||
* Sounds highly natural
|
||||
* Is very fast to synthesise from
|
||||
|
||||
|
||||
Check out our [demo page](https://shivammehta25.github.io/Matcha-TTS). Read our [arXiv preprint for more details](https://arxiv.org/abs/2309.03199).
|
||||
Code is available in our [GitHub repository](https://github.com/shivammehta25/Matcha-TTS), along with pre-trained models.
|
||||
|
||||
Cached examples are available at the bottom of the page.
|
||||
"""
|
||||
|
||||
with gr.Blocks(title="🍵 Matcha-TTS: A fast TTS architecture with conditional flow matching") as demo:
|
||||
processed_text = gr.State(value=None)
|
||||
processed_text_len = gr.State(value=None)
|
||||
|
||||
with gr.Box():
|
||||
with gr.Row():
|
||||
gr.Markdown(description, scale=3)
|
||||
with gr.Column():
|
||||
gr.Image(LOGO_URL, label="Matcha-TTS logo", height=50, width=50, scale=1, show_label=False)
|
||||
html = '<br><iframe width="560" height="315" src="https://www.youtube.com/embed/xmvJkz3bqw0?si=jN7ILyDsbPwJCGoa" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" allowfullscreen></iframe>'
|
||||
gr.HTML(html)
|
||||
|
||||
with gr.Box():
|
||||
radio_options = list(RADIO_OPTIONS.keys())
|
||||
model_type = gr.Radio(
|
||||
radio_options, value=radio_options[0], label="Choose a Model", interactive=True, container=False
|
||||
)
|
||||
|
||||
with gr.Row():
|
||||
gr.Markdown("# Text Input")
|
||||
with gr.Row():
|
||||
text = gr.Textbox(value="", lines=2, label="Text to synthesise", scale=3)
|
||||
spk_slider = gr.Slider(
|
||||
minimum=0, maximum=107, step=1, value=args.spk, label="Speaker ID", interactive=True, scale=1
|
||||
)
|
||||
|
||||
with gr.Row():
|
||||
gr.Markdown("### Hyper parameters")
|
||||
with gr.Row():
|
||||
n_timesteps = gr.Slider(
|
||||
label="Number of ODE steps",
|
||||
minimum=1,
|
||||
maximum=100,
|
||||
step=1,
|
||||
value=10,
|
||||
interactive=True,
|
||||
)
|
||||
length_scale = gr.Slider(
|
||||
label="Length scale (Speaking rate)",
|
||||
minimum=0.5,
|
||||
maximum=1.5,
|
||||
step=0.05,
|
||||
value=1.0,
|
||||
interactive=True,
|
||||
)
|
||||
mel_temp = gr.Slider(
|
||||
label="Sampling temperature",
|
||||
minimum=0.00,
|
||||
maximum=2.001,
|
||||
step=0.16675,
|
||||
value=0.667,
|
||||
interactive=True,
|
||||
)
|
||||
|
||||
synth_btn = gr.Button("Synthesise")
|
||||
|
||||
with gr.Box():
|
||||
with gr.Row():
|
||||
gr.Markdown("### Phonetised text")
|
||||
phonetised_text = gr.Textbox(interactive=False, scale=10, label="Phonetised text")
|
||||
|
||||
with gr.Box():
|
||||
with gr.Row():
|
||||
mel_spectrogram = gr.Image(interactive=False, label="mel spectrogram")
|
||||
|
||||
# with gr.Row():
|
||||
audio = gr.Audio(interactive=False, label="Audio")
|
||||
|
||||
with gr.Row(visible=False) as example_row_lj_speech:
|
||||
examples = gr.Examples( # pylint: disable=unused-variable
|
||||
examples=[
|
||||
[
|
||||
"We propose Matcha-TTS, a new approach to non-autoregressive neural TTS, that uses conditional flow matching (similar to rectified flows) to speed up O D E-based speech synthesis.",
|
||||
50,
|
||||
0.677,
|
||||
0.95,
|
||||
],
|
||||
[
|
||||
"The Secret Service believed that it was very doubtful that any President would ride regularly in a vehicle with a fixed top, even though transparent.",
|
||||
2,
|
||||
0.677,
|
||||
0.95,
|
||||
],
|
||||
[
|
||||
"The Secret Service believed that it was very doubtful that any President would ride regularly in a vehicle with a fixed top, even though transparent.",
|
||||
4,
|
||||
0.677,
|
||||
0.95,
|
||||
],
|
||||
[
|
||||
"The Secret Service believed that it was very doubtful that any President would ride regularly in a vehicle with a fixed top, even though transparent.",
|
||||
10,
|
||||
0.677,
|
||||
0.95,
|
||||
],
|
||||
[
|
||||
"The Secret Service believed that it was very doubtful that any President would ride regularly in a vehicle with a fixed top, even though transparent.",
|
||||
50,
|
||||
0.677,
|
||||
0.95,
|
||||
],
|
||||
[
|
||||
"The narrative of these events is based largely on the recollections of the participants.",
|
||||
10,
|
||||
0.677,
|
||||
0.95,
|
||||
],
|
||||
[
|
||||
"The jury did not believe him, and the verdict was for the defendants.",
|
||||
10,
|
||||
0.677,
|
||||
0.95,
|
||||
],
|
||||
],
|
||||
fn=ljspeech_example_cacher,
|
||||
inputs=[text, n_timesteps, mel_temp, length_scale],
|
||||
outputs=[phonetised_text, audio, mel_spectrogram],
|
||||
cache_examples=True,
|
||||
)
|
||||
|
||||
with gr.Row() as example_row_multispeaker:
|
||||
multi_speaker_examples = gr.Examples( # pylint: disable=unused-variable
|
||||
examples=[
|
||||
[
|
||||
"Hello everyone! I am speaker 0 and I am here to tell you that Matcha-TTS is amazing!",
|
||||
10,
|
||||
0.677,
|
||||
0.85,
|
||||
0,
|
||||
],
|
||||
[
|
||||
"Hello everyone! I am speaker 16 and I am here to tell you that Matcha-TTS is amazing!",
|
||||
10,
|
||||
0.677,
|
||||
0.85,
|
||||
16,
|
||||
],
|
||||
[
|
||||
"Hello everyone! I am speaker 44 and I am here to tell you that Matcha-TTS is amazing!",
|
||||
50,
|
||||
0.677,
|
||||
0.85,
|
||||
44,
|
||||
],
|
||||
[
|
||||
"Hello everyone! I am speaker 45 and I am here to tell you that Matcha-TTS is amazing!",
|
||||
50,
|
||||
0.677,
|
||||
0.85,
|
||||
45,
|
||||
],
|
||||
[
|
||||
"Hello everyone! I am speaker 58 and I am here to tell you that Matcha-TTS is amazing!",
|
||||
4,
|
||||
0.677,
|
||||
0.85,
|
||||
58,
|
||||
],
|
||||
],
|
||||
fn=multispeaker_example_cacher,
|
||||
inputs=[text, n_timesteps, mel_temp, length_scale, spk_slider],
|
||||
outputs=[phonetised_text, audio, mel_spectrogram],
|
||||
cache_examples=True,
|
||||
label="Multi Speaker Examples",
|
||||
)
|
||||
|
||||
model_type.change(lambda x: gr.update(interactive=False), inputs=[synth_btn], outputs=[synth_btn]).then(
|
||||
load_model_ui,
|
||||
inputs=[model_type, text],
|
||||
outputs=[text, synth_btn, spk_slider, example_row_lj_speech, example_row_multispeaker, length_scale],
|
||||
)
|
||||
|
||||
synth_btn.click(
|
||||
fn=process_text_gradio,
|
||||
inputs=[
|
||||
text,
|
||||
],
|
||||
outputs=[phonetised_text, processed_text, processed_text_len],
|
||||
api_name="matcha_tts",
|
||||
queue=True,
|
||||
).then(
|
||||
fn=synthesise_mel,
|
||||
inputs=[processed_text, processed_text_len, n_timesteps, mel_temp, length_scale, spk_slider],
|
||||
outputs=[audio, mel_spectrogram],
|
||||
)
|
||||
|
||||
demo.queue().launch(share=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
418
vendor/CosyVoice/third_party/Matcha-TTS/matcha/cli.py
vendored
Normal file
418
vendor/CosyVoice/third_party/Matcha-TTS/matcha/cli.py
vendored
Normal file
@@ -0,0 +1,418 @@
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import os
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import torch
|
||||
|
||||
from matcha.hifigan.config import v1
|
||||
from matcha.hifigan.denoiser import Denoiser
|
||||
from matcha.hifigan.env import AttrDict
|
||||
from matcha.hifigan.models import Generator as HiFiGAN
|
||||
from matcha.models.matcha_tts import MatchaTTS
|
||||
from matcha.text import sequence_to_text, text_to_sequence
|
||||
from matcha.utils.utils import assert_model_downloaded, get_user_data_dir, intersperse
|
||||
|
||||
MATCHA_URLS = {
|
||||
"matcha_ljspeech": "https://github.com/shivammehta25/Matcha-TTS-checkpoints/releases/download/v1.0/matcha_ljspeech.ckpt",
|
||||
"matcha_vctk": "https://github.com/shivammehta25/Matcha-TTS-checkpoints/releases/download/v1.0/matcha_vctk.ckpt",
|
||||
}
|
||||
|
||||
VOCODER_URLS = {
|
||||
"hifigan_T2_v1": "https://github.com/shivammehta25/Matcha-TTS-checkpoints/releases/download/v1.0/generator_v1", # Old url: https://drive.google.com/file/d/14NENd4equCBLyyCSke114Mv6YR_j_uFs/view?usp=drive_link
|
||||
"hifigan_univ_v1": "https://github.com/shivammehta25/Matcha-TTS-checkpoints/releases/download/v1.0/g_02500000", # Old url: https://drive.google.com/file/d/1qpgI41wNXFcH-iKq1Y42JlBC9j0je8PW/view?usp=drive_link
|
||||
}
|
||||
|
||||
MULTISPEAKER_MODEL = {
|
||||
"matcha_vctk": {"vocoder": "hifigan_univ_v1", "speaking_rate": 0.85, "spk": 0, "spk_range": (0, 107)}
|
||||
}
|
||||
|
||||
SINGLESPEAKER_MODEL = {"matcha_ljspeech": {"vocoder": "hifigan_T2_v1", "speaking_rate": 0.95, "spk": None}}
|
||||
|
||||
|
||||
def plot_spectrogram_to_numpy(spectrogram, filename):
|
||||
fig, ax = plt.subplots(figsize=(12, 3))
|
||||
im = ax.imshow(spectrogram, aspect="auto", origin="lower", interpolation="none")
|
||||
plt.colorbar(im, ax=ax)
|
||||
plt.xlabel("Frames")
|
||||
plt.ylabel("Channels")
|
||||
plt.title("Synthesised Mel-Spectrogram")
|
||||
fig.canvas.draw()
|
||||
plt.savefig(filename)
|
||||
|
||||
|
||||
def process_text(i: int, text: str, device: torch.device):
|
||||
print(f"[{i}] - Input text: {text}")
|
||||
x = torch.tensor(
|
||||
intersperse(text_to_sequence(text, ["english_cleaners2"]), 0),
|
||||
dtype=torch.long,
|
||||
device=device,
|
||||
)[None]
|
||||
x_lengths = torch.tensor([x.shape[-1]], dtype=torch.long, device=device)
|
||||
x_phones = sequence_to_text(x.squeeze(0).tolist())
|
||||
print(f"[{i}] - Phonetised text: {x_phones[1::2]}")
|
||||
|
||||
return {"x_orig": text, "x": x, "x_lengths": x_lengths, "x_phones": x_phones}
|
||||
|
||||
|
||||
def get_texts(args):
|
||||
if args.text:
|
||||
texts = [args.text]
|
||||
else:
|
||||
with open(args.file, encoding="utf-8") as f:
|
||||
texts = f.readlines()
|
||||
return texts
|
||||
|
||||
|
||||
def assert_required_models_available(args):
|
||||
save_dir = get_user_data_dir()
|
||||
if not hasattr(args, "checkpoint_path") and args.checkpoint_path is None:
|
||||
model_path = args.checkpoint_path
|
||||
else:
|
||||
model_path = save_dir / f"{args.model}.ckpt"
|
||||
assert_model_downloaded(model_path, MATCHA_URLS[args.model])
|
||||
|
||||
vocoder_path = save_dir / f"{args.vocoder}"
|
||||
assert_model_downloaded(vocoder_path, VOCODER_URLS[args.vocoder])
|
||||
return {"matcha": model_path, "vocoder": vocoder_path}
|
||||
|
||||
|
||||
def load_hifigan(checkpoint_path, device):
|
||||
h = AttrDict(v1)
|
||||
hifigan = HiFiGAN(h).to(device)
|
||||
hifigan.load_state_dict(torch.load(checkpoint_path, map_location=device)["generator"])
|
||||
_ = hifigan.eval()
|
||||
hifigan.remove_weight_norm()
|
||||
return hifigan
|
||||
|
||||
|
||||
def load_vocoder(vocoder_name, checkpoint_path, device):
|
||||
print(f"[!] Loading {vocoder_name}!")
|
||||
vocoder = None
|
||||
if vocoder_name in ("hifigan_T2_v1", "hifigan_univ_v1"):
|
||||
vocoder = load_hifigan(checkpoint_path, device)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"Vocoder {vocoder_name} not implemented! define a load_<<vocoder_name>> method for it"
|
||||
)
|
||||
|
||||
denoiser = Denoiser(vocoder, mode="zeros")
|
||||
print(f"[+] {vocoder_name} loaded!")
|
||||
return vocoder, denoiser
|
||||
|
||||
|
||||
def load_matcha(model_name, checkpoint_path, device):
|
||||
print(f"[!] Loading {model_name}!")
|
||||
model = MatchaTTS.load_from_checkpoint(checkpoint_path, map_location=device)
|
||||
_ = model.eval()
|
||||
|
||||
print(f"[+] {model_name} loaded!")
|
||||
return model
|
||||
|
||||
|
||||
def to_waveform(mel, vocoder, denoiser=None):
|
||||
audio = vocoder(mel).clamp(-1, 1)
|
||||
if denoiser is not None:
|
||||
audio = denoiser(audio.squeeze(), strength=0.00025).cpu().squeeze()
|
||||
|
||||
return audio.cpu().squeeze()
|
||||
|
||||
|
||||
def save_to_folder(filename: str, output: dict, folder: str):
|
||||
folder = Path(folder)
|
||||
folder.mkdir(exist_ok=True, parents=True)
|
||||
plot_spectrogram_to_numpy(np.array(output["mel"].squeeze().float().cpu()), f"{filename}.png")
|
||||
np.save(folder / f"{filename}", output["mel"].cpu().numpy())
|
||||
sf.write(folder / f"{filename}.wav", output["waveform"], 22050, "PCM_24")
|
||||
return folder.resolve() / f"{filename}.wav"
|
||||
|
||||
|
||||
def validate_args(args):
|
||||
assert (
|
||||
args.text or args.file
|
||||
), "Either text or file must be provided Matcha-T(ea)TTS need sometext to whisk the waveforms."
|
||||
assert args.temperature >= 0, "Sampling temperature cannot be negative"
|
||||
assert args.steps > 0, "Number of ODE steps must be greater than 0"
|
||||
|
||||
if args.checkpoint_path is None:
|
||||
# When using pretrained models
|
||||
if args.model in SINGLESPEAKER_MODEL:
|
||||
args = validate_args_for_single_speaker_model(args)
|
||||
|
||||
if args.model in MULTISPEAKER_MODEL:
|
||||
args = validate_args_for_multispeaker_model(args)
|
||||
else:
|
||||
# When using a custom model
|
||||
if args.vocoder != "hifigan_univ_v1":
|
||||
warn_ = "[-] Using custom model checkpoint! I would suggest passing --vocoder hifigan_univ_v1, unless the custom model is trained on LJ Speech."
|
||||
warnings.warn(warn_, UserWarning)
|
||||
if args.speaking_rate is None:
|
||||
args.speaking_rate = 1.0
|
||||
|
||||
if args.batched:
|
||||
assert args.batch_size > 0, "Batch size must be greater than 0"
|
||||
assert args.speaking_rate > 0, "Speaking rate must be greater than 0"
|
||||
|
||||
return args
|
||||
|
||||
|
||||
def validate_args_for_multispeaker_model(args):
|
||||
if args.vocoder is not None:
|
||||
if args.vocoder != MULTISPEAKER_MODEL[args.model]["vocoder"]:
|
||||
warn_ = f"[-] Using {args.model} model! I would suggest passing --vocoder {MULTISPEAKER_MODEL[args.model]['vocoder']}"
|
||||
warnings.warn(warn_, UserWarning)
|
||||
else:
|
||||
args.vocoder = MULTISPEAKER_MODEL[args.model]["vocoder"]
|
||||
|
||||
if args.speaking_rate is None:
|
||||
args.speaking_rate = MULTISPEAKER_MODEL[args.model]["speaking_rate"]
|
||||
|
||||
spk_range = MULTISPEAKER_MODEL[args.model]["spk_range"]
|
||||
if args.spk is not None:
|
||||
assert (
|
||||
args.spk >= spk_range[0] and args.spk <= spk_range[-1]
|
||||
), f"Speaker ID must be between {spk_range} for this model."
|
||||
else:
|
||||
available_spk_id = MULTISPEAKER_MODEL[args.model]["spk"]
|
||||
warn_ = f"[!] Speaker ID not provided! Using speaker ID {available_spk_id}"
|
||||
warnings.warn(warn_, UserWarning)
|
||||
args.spk = available_spk_id
|
||||
|
||||
return args
|
||||
|
||||
|
||||
def validate_args_for_single_speaker_model(args):
|
||||
if args.vocoder is not None:
|
||||
if args.vocoder != SINGLESPEAKER_MODEL[args.model]["vocoder"]:
|
||||
warn_ = f"[-] Using {args.model} model! I would suggest passing --vocoder {SINGLESPEAKER_MODEL[args.model]['vocoder']}"
|
||||
warnings.warn(warn_, UserWarning)
|
||||
else:
|
||||
args.vocoder = SINGLESPEAKER_MODEL[args.model]["vocoder"]
|
||||
|
||||
if args.speaking_rate is None:
|
||||
args.speaking_rate = SINGLESPEAKER_MODEL[args.model]["speaking_rate"]
|
||||
|
||||
if args.spk != SINGLESPEAKER_MODEL[args.model]["spk"]:
|
||||
warn_ = f"[-] Ignoring speaker id {args.spk} for {args.model}"
|
||||
warnings.warn(warn_, UserWarning)
|
||||
args.spk = SINGLESPEAKER_MODEL[args.model]["spk"]
|
||||
|
||||
return args
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def cli():
|
||||
parser = argparse.ArgumentParser(
|
||||
description=" 🍵 Matcha-TTS: A fast TTS architecture with conditional flow matching"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
default="matcha_ljspeech",
|
||||
help="Model to use",
|
||||
choices=MATCHA_URLS.keys(),
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--checkpoint_path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to the custom model checkpoint",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--vocoder",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Vocoder to use (default: will use the one suggested with the pretrained model))",
|
||||
choices=VOCODER_URLS.keys(),
|
||||
)
|
||||
parser.add_argument("--text", type=str, default=None, help="Text to synthesize")
|
||||
parser.add_argument("--file", type=str, default=None, help="Text file to synthesize")
|
||||
parser.add_argument("--spk", type=int, default=None, help="Speaker ID")
|
||||
parser.add_argument(
|
||||
"--temperature",
|
||||
type=float,
|
||||
default=0.667,
|
||||
help="Variance of the x0 noise (default: 0.667)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--speaking_rate",
|
||||
type=float,
|
||||
default=None,
|
||||
help="change the speaking rate, a higher value means slower speaking rate (default: 1.0)",
|
||||
)
|
||||
parser.add_argument("--steps", type=int, default=10, help="Number of ODE steps (default: 10)")
|
||||
parser.add_argument("--cpu", action="store_true", help="Use CPU for inference (default: use GPU if available)")
|
||||
parser.add_argument(
|
||||
"--denoiser_strength",
|
||||
type=float,
|
||||
default=0.00025,
|
||||
help="Strength of the vocoder bias denoiser (default: 0.00025)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output_folder",
|
||||
type=str,
|
||||
default=os.getcwd(),
|
||||
help="Output folder to save results (default: current dir)",
|
||||
)
|
||||
parser.add_argument("--batched", action="store_true", help="Batched inference (default: False)")
|
||||
parser.add_argument(
|
||||
"--batch_size", type=int, default=32, help="Batch size only useful when --batched (default: 32)"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
args = validate_args(args)
|
||||
device = get_device(args)
|
||||
print_config(args)
|
||||
paths = assert_required_models_available(args)
|
||||
|
||||
if args.checkpoint_path is not None:
|
||||
print(f"[🍵] Loading custom model from {args.checkpoint_path}")
|
||||
paths["matcha"] = args.checkpoint_path
|
||||
args.model = "custom_model"
|
||||
|
||||
model = load_matcha(args.model, paths["matcha"], device)
|
||||
vocoder, denoiser = load_vocoder(args.vocoder, paths["vocoder"], device)
|
||||
|
||||
texts = get_texts(args)
|
||||
|
||||
spk = torch.tensor([args.spk], device=device, dtype=torch.long) if args.spk is not None else None
|
||||
if len(texts) == 1 or not args.batched:
|
||||
unbatched_synthesis(args, device, model, vocoder, denoiser, texts, spk)
|
||||
else:
|
||||
batched_synthesis(args, device, model, vocoder, denoiser, texts, spk)
|
||||
|
||||
|
||||
class BatchedSynthesisDataset(torch.utils.data.Dataset):
|
||||
def __init__(self, processed_texts):
|
||||
self.processed_texts = processed_texts
|
||||
|
||||
def __len__(self):
|
||||
return len(self.processed_texts)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
return self.processed_texts[idx]
|
||||
|
||||
|
||||
def batched_collate_fn(batch):
|
||||
x = []
|
||||
x_lengths = []
|
||||
|
||||
for b in batch:
|
||||
x.append(b["x"].squeeze(0))
|
||||
x_lengths.append(b["x_lengths"])
|
||||
|
||||
x = torch.nn.utils.rnn.pad_sequence(x, batch_first=True)
|
||||
x_lengths = torch.concat(x_lengths, dim=0)
|
||||
return {"x": x, "x_lengths": x_lengths}
|
||||
|
||||
|
||||
def batched_synthesis(args, device, model, vocoder, denoiser, texts, spk):
|
||||
total_rtf = []
|
||||
total_rtf_w = []
|
||||
processed_text = [process_text(i, text, "cpu") for i, text in enumerate(texts)]
|
||||
dataloader = torch.utils.data.DataLoader(
|
||||
BatchedSynthesisDataset(processed_text),
|
||||
batch_size=args.batch_size,
|
||||
collate_fn=batched_collate_fn,
|
||||
num_workers=8,
|
||||
)
|
||||
for i, batch in enumerate(dataloader):
|
||||
i = i + 1
|
||||
start_t = dt.datetime.now()
|
||||
output = model.synthesise(
|
||||
batch["x"].to(device),
|
||||
batch["x_lengths"].to(device),
|
||||
n_timesteps=args.steps,
|
||||
temperature=args.temperature,
|
||||
spks=spk,
|
||||
length_scale=args.speaking_rate,
|
||||
)
|
||||
|
||||
output["waveform"] = to_waveform(output["mel"], vocoder, denoiser)
|
||||
t = (dt.datetime.now() - start_t).total_seconds()
|
||||
rtf_w = t * 22050 / (output["waveform"].shape[-1])
|
||||
print(f"[🍵-Batch: {i}] Matcha-TTS RTF: {output['rtf']:.4f}")
|
||||
print(f"[🍵-Batch: {i}] Matcha-TTS + VOCODER RTF: {rtf_w:.4f}")
|
||||
total_rtf.append(output["rtf"])
|
||||
total_rtf_w.append(rtf_w)
|
||||
for j in range(output["mel"].shape[0]):
|
||||
base_name = f"utterance_{j:03d}_speaker_{args.spk:03d}" if args.spk is not None else f"utterance_{j:03d}"
|
||||
length = output["mel_lengths"][j]
|
||||
new_dict = {"mel": output["mel"][j][:, :length], "waveform": output["waveform"][j][: length * 256]}
|
||||
location = save_to_folder(base_name, new_dict, args.output_folder)
|
||||
print(f"[🍵-{j}] Waveform saved: {location}")
|
||||
|
||||
print("".join(["="] * 100))
|
||||
print(f"[🍵] Average Matcha-TTS RTF: {np.mean(total_rtf):.4f} ± {np.std(total_rtf)}")
|
||||
print(f"[🍵] Average Matcha-TTS + VOCODER RTF: {np.mean(total_rtf_w):.4f} ± {np.std(total_rtf_w)}")
|
||||
print("[🍵] Enjoy the freshly whisked 🍵 Matcha-TTS!")
|
||||
|
||||
|
||||
def unbatched_synthesis(args, device, model, vocoder, denoiser, texts, spk):
|
||||
total_rtf = []
|
||||
total_rtf_w = []
|
||||
for i, text in enumerate(texts):
|
||||
i = i + 1
|
||||
base_name = f"utterance_{i:03d}_speaker_{args.spk:03d}" if args.spk is not None else f"utterance_{i:03d}"
|
||||
|
||||
print("".join(["="] * 100))
|
||||
text = text.strip()
|
||||
text_processed = process_text(i, text, device)
|
||||
|
||||
print(f"[🍵] Whisking Matcha-T(ea)TS for: {i}")
|
||||
start_t = dt.datetime.now()
|
||||
output = model.synthesise(
|
||||
text_processed["x"],
|
||||
text_processed["x_lengths"],
|
||||
n_timesteps=args.steps,
|
||||
temperature=args.temperature,
|
||||
spks=spk,
|
||||
length_scale=args.speaking_rate,
|
||||
)
|
||||
output["waveform"] = to_waveform(output["mel"], vocoder, denoiser)
|
||||
# RTF with HiFiGAN
|
||||
t = (dt.datetime.now() - start_t).total_seconds()
|
||||
rtf_w = t * 22050 / (output["waveform"].shape[-1])
|
||||
print(f"[🍵-{i}] Matcha-TTS RTF: {output['rtf']:.4f}")
|
||||
print(f"[🍵-{i}] Matcha-TTS + VOCODER RTF: {rtf_w:.4f}")
|
||||
total_rtf.append(output["rtf"])
|
||||
total_rtf_w.append(rtf_w)
|
||||
|
||||
location = save_to_folder(base_name, output, args.output_folder)
|
||||
print(f"[+] Waveform saved: {location}")
|
||||
|
||||
print("".join(["="] * 100))
|
||||
print(f"[🍵] Average Matcha-TTS RTF: {np.mean(total_rtf):.4f} ± {np.std(total_rtf)}")
|
||||
print(f"[🍵] Average Matcha-TTS + VOCODER RTF: {np.mean(total_rtf_w):.4f} ± {np.std(total_rtf_w)}")
|
||||
print("[🍵] Enjoy the freshly whisked 🍵 Matcha-TTS!")
|
||||
|
||||
|
||||
def print_config(args):
|
||||
print("[!] Configurations: ")
|
||||
print(f"\t- Model: {args.model}")
|
||||
print(f"\t- Vocoder: {args.vocoder}")
|
||||
print(f"\t- Temperature: {args.temperature}")
|
||||
print(f"\t- Speaking rate: {args.speaking_rate}")
|
||||
print(f"\t- Number of ODE steps: {args.steps}")
|
||||
print(f"\t- Speaker: {args.spk}")
|
||||
|
||||
|
||||
def get_device(args):
|
||||
if torch.cuda.is_available() and not args.cpu:
|
||||
print("[+] GPU Available! Using GPU")
|
||||
device = torch.device("cuda")
|
||||
else:
|
||||
print("[-] GPU not available or forced CPU run! Using CPU")
|
||||
device = torch.device("cpu")
|
||||
return device
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
0
vendor/CosyVoice/third_party/Matcha-TTS/matcha/data/__init__.py
vendored
Normal file
0
vendor/CosyVoice/third_party/Matcha-TTS/matcha/data/__init__.py
vendored
Normal file
0
vendor/CosyVoice/third_party/Matcha-TTS/matcha/data/components/__init__.py
vendored
Normal file
0
vendor/CosyVoice/third_party/Matcha-TTS/matcha/data/components/__init__.py
vendored
Normal file
231
vendor/CosyVoice/third_party/Matcha-TTS/matcha/data/text_mel_datamodule.py
vendored
Normal file
231
vendor/CosyVoice/third_party/Matcha-TTS/matcha/data/text_mel_datamodule.py
vendored
Normal file
@@ -0,0 +1,231 @@
|
||||
import random
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import torch
|
||||
import torchaudio as ta
|
||||
from lightning import LightningDataModule
|
||||
from torch.utils.data.dataloader import DataLoader
|
||||
|
||||
from matcha.text import text_to_sequence
|
||||
from matcha.utils.audio import mel_spectrogram
|
||||
from matcha.utils.model import fix_len_compatibility, normalize
|
||||
from matcha.utils.utils import intersperse
|
||||
|
||||
|
||||
def parse_filelist(filelist_path, split_char="|"):
|
||||
with open(filelist_path, encoding="utf-8") as f:
|
||||
filepaths_and_text = [line.strip().split(split_char) for line in f]
|
||||
return filepaths_and_text
|
||||
|
||||
|
||||
class TextMelDataModule(LightningDataModule):
|
||||
def __init__( # pylint: disable=unused-argument
|
||||
self,
|
||||
name,
|
||||
train_filelist_path,
|
||||
valid_filelist_path,
|
||||
batch_size,
|
||||
num_workers,
|
||||
pin_memory,
|
||||
cleaners,
|
||||
add_blank,
|
||||
n_spks,
|
||||
n_fft,
|
||||
n_feats,
|
||||
sample_rate,
|
||||
hop_length,
|
||||
win_length,
|
||||
f_min,
|
||||
f_max,
|
||||
data_statistics,
|
||||
seed,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
# this line allows to access init params with 'self.hparams' attribute
|
||||
# also ensures init params will be stored in ckpt
|
||||
self.save_hyperparameters(logger=False)
|
||||
|
||||
def setup(self, stage: Optional[str] = None): # pylint: disable=unused-argument
|
||||
"""Load data. Set variables: `self.data_train`, `self.data_val`, `self.data_test`.
|
||||
|
||||
This method is called by lightning with both `trainer.fit()` and `trainer.test()`, so be
|
||||
careful not to execute things like random split twice!
|
||||
"""
|
||||
# load and split datasets only if not loaded already
|
||||
|
||||
self.trainset = TextMelDataset( # pylint: disable=attribute-defined-outside-init
|
||||
self.hparams.train_filelist_path,
|
||||
self.hparams.n_spks,
|
||||
self.hparams.cleaners,
|
||||
self.hparams.add_blank,
|
||||
self.hparams.n_fft,
|
||||
self.hparams.n_feats,
|
||||
self.hparams.sample_rate,
|
||||
self.hparams.hop_length,
|
||||
self.hparams.win_length,
|
||||
self.hparams.f_min,
|
||||
self.hparams.f_max,
|
||||
self.hparams.data_statistics,
|
||||
self.hparams.seed,
|
||||
)
|
||||
self.validset = TextMelDataset( # pylint: disable=attribute-defined-outside-init
|
||||
self.hparams.valid_filelist_path,
|
||||
self.hparams.n_spks,
|
||||
self.hparams.cleaners,
|
||||
self.hparams.add_blank,
|
||||
self.hparams.n_fft,
|
||||
self.hparams.n_feats,
|
||||
self.hparams.sample_rate,
|
||||
self.hparams.hop_length,
|
||||
self.hparams.win_length,
|
||||
self.hparams.f_min,
|
||||
self.hparams.f_max,
|
||||
self.hparams.data_statistics,
|
||||
self.hparams.seed,
|
||||
)
|
||||
|
||||
def train_dataloader(self):
|
||||
return DataLoader(
|
||||
dataset=self.trainset,
|
||||
batch_size=self.hparams.batch_size,
|
||||
num_workers=self.hparams.num_workers,
|
||||
pin_memory=self.hparams.pin_memory,
|
||||
shuffle=True,
|
||||
collate_fn=TextMelBatchCollate(self.hparams.n_spks),
|
||||
)
|
||||
|
||||
def val_dataloader(self):
|
||||
return DataLoader(
|
||||
dataset=self.validset,
|
||||
batch_size=self.hparams.batch_size,
|
||||
num_workers=self.hparams.num_workers,
|
||||
pin_memory=self.hparams.pin_memory,
|
||||
shuffle=False,
|
||||
collate_fn=TextMelBatchCollate(self.hparams.n_spks),
|
||||
)
|
||||
|
||||
def teardown(self, stage: Optional[str] = None):
|
||||
"""Clean up after fit or test."""
|
||||
pass # pylint: disable=unnecessary-pass
|
||||
|
||||
def state_dict(self): # pylint: disable=no-self-use
|
||||
"""Extra things to save to checkpoint."""
|
||||
return {}
|
||||
|
||||
def load_state_dict(self, state_dict: Dict[str, Any]):
|
||||
"""Things to do when loading checkpoint."""
|
||||
pass # pylint: disable=unnecessary-pass
|
||||
|
||||
|
||||
class TextMelDataset(torch.utils.data.Dataset):
|
||||
def __init__(
|
||||
self,
|
||||
filelist_path,
|
||||
n_spks,
|
||||
cleaners,
|
||||
add_blank=True,
|
||||
n_fft=1024,
|
||||
n_mels=80,
|
||||
sample_rate=22050,
|
||||
hop_length=256,
|
||||
win_length=1024,
|
||||
f_min=0.0,
|
||||
f_max=8000,
|
||||
data_parameters=None,
|
||||
seed=None,
|
||||
):
|
||||
self.filepaths_and_text = parse_filelist(filelist_path)
|
||||
self.n_spks = n_spks
|
||||
self.cleaners = cleaners
|
||||
self.add_blank = add_blank
|
||||
self.n_fft = n_fft
|
||||
self.n_mels = n_mels
|
||||
self.sample_rate = sample_rate
|
||||
self.hop_length = hop_length
|
||||
self.win_length = win_length
|
||||
self.f_min = f_min
|
||||
self.f_max = f_max
|
||||
if data_parameters is not None:
|
||||
self.data_parameters = data_parameters
|
||||
else:
|
||||
self.data_parameters = {"mel_mean": 0, "mel_std": 1}
|
||||
random.seed(seed)
|
||||
random.shuffle(self.filepaths_and_text)
|
||||
|
||||
def get_datapoint(self, filepath_and_text):
|
||||
if self.n_spks > 1:
|
||||
filepath, spk, text = (
|
||||
filepath_and_text[0],
|
||||
int(filepath_and_text[1]),
|
||||
filepath_and_text[2],
|
||||
)
|
||||
else:
|
||||
filepath, text = filepath_and_text[0], filepath_and_text[1]
|
||||
spk = None
|
||||
|
||||
text = self.get_text(text, add_blank=self.add_blank)
|
||||
mel = self.get_mel(filepath)
|
||||
|
||||
return {"x": text, "y": mel, "spk": spk}
|
||||
|
||||
def get_mel(self, filepath):
|
||||
audio, sr = ta.load(filepath)
|
||||
assert sr == self.sample_rate
|
||||
mel = mel_spectrogram(
|
||||
audio,
|
||||
self.n_fft,
|
||||
self.n_mels,
|
||||
self.sample_rate,
|
||||
self.hop_length,
|
||||
self.win_length,
|
||||
self.f_min,
|
||||
self.f_max,
|
||||
center=False,
|
||||
).squeeze()
|
||||
mel = normalize(mel, self.data_parameters["mel_mean"], self.data_parameters["mel_std"])
|
||||
return mel
|
||||
|
||||
def get_text(self, text, add_blank=True):
|
||||
text_norm = text_to_sequence(text, self.cleaners)
|
||||
if self.add_blank:
|
||||
text_norm = intersperse(text_norm, 0)
|
||||
text_norm = torch.IntTensor(text_norm)
|
||||
return text_norm
|
||||
|
||||
def __getitem__(self, index):
|
||||
datapoint = self.get_datapoint(self.filepaths_and_text[index])
|
||||
return datapoint
|
||||
|
||||
def __len__(self):
|
||||
return len(self.filepaths_and_text)
|
||||
|
||||
|
||||
class TextMelBatchCollate:
|
||||
def __init__(self, n_spks):
|
||||
self.n_spks = n_spks
|
||||
|
||||
def __call__(self, batch):
|
||||
B = len(batch)
|
||||
y_max_length = max([item["y"].shape[-1] for item in batch])
|
||||
y_max_length = fix_len_compatibility(y_max_length)
|
||||
x_max_length = max([item["x"].shape[-1] for item in batch])
|
||||
n_feats = batch[0]["y"].shape[-2]
|
||||
|
||||
y = torch.zeros((B, n_feats, y_max_length), dtype=torch.float32)
|
||||
x = torch.zeros((B, x_max_length), dtype=torch.long)
|
||||
y_lengths, x_lengths = [], []
|
||||
spks = []
|
||||
for i, item in enumerate(batch):
|
||||
y_, x_ = item["y"], item["x"]
|
||||
y_lengths.append(y_.shape[-1])
|
||||
x_lengths.append(x_.shape[-1])
|
||||
y[i, :, : y_.shape[-1]] = y_
|
||||
x[i, : x_.shape[-1]] = x_
|
||||
spks.append(item["spk"])
|
||||
|
||||
y_lengths = torch.tensor(y_lengths, dtype=torch.long)
|
||||
x_lengths = torch.tensor(x_lengths, dtype=torch.long)
|
||||
spks = torch.tensor(spks, dtype=torch.long) if self.n_spks > 1 else None
|
||||
|
||||
return {"x": x, "x_lengths": x_lengths, "y": y, "y_lengths": y_lengths, "spks": spks}
|
||||
21
vendor/CosyVoice/third_party/Matcha-TTS/matcha/hifigan/LICENSE
vendored
Normal file
21
vendor/CosyVoice/third_party/Matcha-TTS/matcha/hifigan/LICENSE
vendored
Normal file
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2020 Jungil Kong
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
101
vendor/CosyVoice/third_party/Matcha-TTS/matcha/hifigan/README.md
vendored
Normal file
101
vendor/CosyVoice/third_party/Matcha-TTS/matcha/hifigan/README.md
vendored
Normal file
@@ -0,0 +1,101 @@
|
||||
# HiFi-GAN: Generative Adversarial Networks for Efficient and High Fidelity Speech Synthesis
|
||||
|
||||
### Jungil Kong, Jaehyeon Kim, Jaekyoung Bae
|
||||
|
||||
In our [paper](https://arxiv.org/abs/2010.05646),
|
||||
we proposed HiFi-GAN: a GAN-based model capable of generating high fidelity speech efficiently.<br/>
|
||||
We provide our implementation and pretrained models as open source in this repository.
|
||||
|
||||
**Abstract :**
|
||||
Several recent work on speech synthesis have employed generative adversarial networks (GANs) to produce raw waveforms.
|
||||
Although such methods improve the sampling efficiency and memory usage,
|
||||
their sample quality has not yet reached that of autoregressive and flow-based generative models.
|
||||
In this work, we propose HiFi-GAN, which achieves both efficient and high-fidelity speech synthesis.
|
||||
As speech audio consists of sinusoidal signals with various periods,
|
||||
we demonstrate that modeling periodic patterns of an audio is crucial for enhancing sample quality.
|
||||
A subjective human evaluation (mean opinion score, MOS) of a single speaker dataset indicates that our proposed method
|
||||
demonstrates similarity to human quality while generating 22.05 kHz high-fidelity audio 167.9 times faster than
|
||||
real-time on a single V100 GPU. We further show the generality of HiFi-GAN to the mel-spectrogram inversion of unseen
|
||||
speakers and end-to-end speech synthesis. Finally, a small footprint version of HiFi-GAN generates samples 13.4 times
|
||||
faster than real-time on CPU with comparable quality to an autoregressive counterpart.
|
||||
|
||||
Visit our [demo website](https://jik876.github.io/hifi-gan-demo/) for audio samples.
|
||||
|
||||
## Pre-requisites
|
||||
|
||||
1. Python >= 3.6
|
||||
2. Clone this repository.
|
||||
3. Install python requirements. Please refer [requirements.txt](requirements.txt)
|
||||
4. Download and extract the [LJ Speech dataset](https://keithito.com/LJ-Speech-Dataset/).
|
||||
And move all wav files to `LJSpeech-1.1/wavs`
|
||||
|
||||
## Training
|
||||
|
||||
```
|
||||
python train.py --config config_v1.json
|
||||
```
|
||||
|
||||
To train V2 or V3 Generator, replace `config_v1.json` with `config_v2.json` or `config_v3.json`.<br>
|
||||
Checkpoints and copy of the configuration file are saved in `cp_hifigan` directory by default.<br>
|
||||
You can change the path by adding `--checkpoint_path` option.
|
||||
|
||||
Validation loss during training with V1 generator.<br>
|
||||
|
||||
|
||||
## Pretrained Model
|
||||
|
||||
You can also use pretrained models we provide.<br/>
|
||||
[Download pretrained models](https://drive.google.com/drive/folders/1-eEYTB5Av9jNql0WGBlRoi-WH2J7bp5Y?usp=sharing)<br/>
|
||||
Details of each folder are as in follows:
|
||||
|
||||
| Folder Name | Generator | Dataset | Fine-Tuned |
|
||||
| ------------ | --------- | --------- | ------------------------------------------------------ |
|
||||
| LJ_V1 | V1 | LJSpeech | No |
|
||||
| LJ_V2 | V2 | LJSpeech | No |
|
||||
| LJ_V3 | V3 | LJSpeech | No |
|
||||
| LJ_FT_T2_V1 | V1 | LJSpeech | Yes ([Tacotron2](https://github.com/NVIDIA/tacotron2)) |
|
||||
| LJ_FT_T2_V2 | V2 | LJSpeech | Yes ([Tacotron2](https://github.com/NVIDIA/tacotron2)) |
|
||||
| LJ_FT_T2_V3 | V3 | LJSpeech | Yes ([Tacotron2](https://github.com/NVIDIA/tacotron2)) |
|
||||
| VCTK_V1 | V1 | VCTK | No |
|
||||
| VCTK_V2 | V2 | VCTK | No |
|
||||
| VCTK_V3 | V3 | VCTK | No |
|
||||
| UNIVERSAL_V1 | V1 | Universal | No |
|
||||
|
||||
We provide the universal model with discriminator weights that can be used as a base for transfer learning to other datasets.
|
||||
|
||||
## Fine-Tuning
|
||||
|
||||
1. Generate mel-spectrograms in numpy format using [Tacotron2](https://github.com/NVIDIA/tacotron2) with teacher-forcing.<br/>
|
||||
The file name of the generated mel-spectrogram should match the audio file and the extension should be `.npy`.<br/>
|
||||
Example:
|
||||
` Audio File : LJ001-0001.wav
|
||||
Mel-Spectrogram File : LJ001-0001.npy`
|
||||
2. Create `ft_dataset` folder and copy the generated mel-spectrogram files into it.<br/>
|
||||
3. Run the following command.
|
||||
```
|
||||
python train.py --fine_tuning True --config config_v1.json
|
||||
```
|
||||
For other command line options, please refer to the training section.
|
||||
|
||||
## Inference from wav file
|
||||
|
||||
1. Make `test_files` directory and copy wav files into the directory.
|
||||
2. Run the following command.
|
||||
` python inference.py --checkpoint_file [generator checkpoint file path]`
|
||||
Generated wav files are saved in `generated_files` by default.<br>
|
||||
You can change the path by adding `--output_dir` option.
|
||||
|
||||
## Inference for end-to-end speech synthesis
|
||||
|
||||
1. Make `test_mel_files` directory and copy generated mel-spectrogram files into the directory.<br>
|
||||
You can generate mel-spectrograms using [Tacotron2](https://github.com/NVIDIA/tacotron2),
|
||||
[Glow-TTS](https://github.com/jaywalnut310/glow-tts) and so forth.
|
||||
2. Run the following command.
|
||||
` python inference_e2e.py --checkpoint_file [generator checkpoint file path]`
|
||||
Generated wav files are saved in `generated_files_from_mel` by default.<br>
|
||||
You can change the path by adding `--output_dir` option.
|
||||
|
||||
## Acknowledgements
|
||||
|
||||
We referred to [WaveGlow](https://github.com/NVIDIA/waveglow), [MelGAN](https://github.com/descriptinc/melgan-neurips)
|
||||
and [Tacotron2](https://github.com/NVIDIA/tacotron2) to implement this.
|
||||
0
vendor/CosyVoice/third_party/Matcha-TTS/matcha/hifigan/__init__.py
vendored
Normal file
0
vendor/CosyVoice/third_party/Matcha-TTS/matcha/hifigan/__init__.py
vendored
Normal file
28
vendor/CosyVoice/third_party/Matcha-TTS/matcha/hifigan/config.py
vendored
Normal file
28
vendor/CosyVoice/third_party/Matcha-TTS/matcha/hifigan/config.py
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
v1 = {
|
||||
"resblock": "1",
|
||||
"num_gpus": 0,
|
||||
"batch_size": 16,
|
||||
"learning_rate": 0.0004,
|
||||
"adam_b1": 0.8,
|
||||
"adam_b2": 0.99,
|
||||
"lr_decay": 0.999,
|
||||
"seed": 1234,
|
||||
"upsample_rates": [8, 8, 2, 2],
|
||||
"upsample_kernel_sizes": [16, 16, 4, 4],
|
||||
"upsample_initial_channel": 512,
|
||||
"resblock_kernel_sizes": [3, 7, 11],
|
||||
"resblock_dilation_sizes": [[1, 3, 5], [1, 3, 5], [1, 3, 5]],
|
||||
"resblock_initial_channel": 256,
|
||||
"segment_size": 8192,
|
||||
"num_mels": 80,
|
||||
"num_freq": 1025,
|
||||
"n_fft": 1024,
|
||||
"hop_size": 256,
|
||||
"win_size": 1024,
|
||||
"sampling_rate": 22050,
|
||||
"fmin": 0,
|
||||
"fmax": 8000,
|
||||
"fmax_loss": None,
|
||||
"num_workers": 4,
|
||||
"dist_config": {"dist_backend": "nccl", "dist_url": "tcp://localhost:54321", "world_size": 1},
|
||||
}
|
||||
64
vendor/CosyVoice/third_party/Matcha-TTS/matcha/hifigan/denoiser.py
vendored
Normal file
64
vendor/CosyVoice/third_party/Matcha-TTS/matcha/hifigan/denoiser.py
vendored
Normal file
@@ -0,0 +1,64 @@
|
||||
# Code modified from Rafael Valle's implementation https://github.com/NVIDIA/waveglow/blob/5bc2a53e20b3b533362f974cfa1ea0267ae1c2b1/denoiser.py
|
||||
|
||||
"""Waveglow style denoiser can be used to remove the artifacts from the HiFiGAN generated audio."""
|
||||
import torch
|
||||
|
||||
|
||||
class Denoiser(torch.nn.Module):
|
||||
"""Removes model bias from audio produced with waveglow"""
|
||||
|
||||
def __init__(self, vocoder, filter_length=1024, n_overlap=4, win_length=1024, mode="zeros"):
|
||||
super().__init__()
|
||||
self.filter_length = filter_length
|
||||
self.hop_length = int(filter_length / n_overlap)
|
||||
self.win_length = win_length
|
||||
|
||||
dtype, device = next(vocoder.parameters()).dtype, next(vocoder.parameters()).device
|
||||
self.device = device
|
||||
if mode == "zeros":
|
||||
mel_input = torch.zeros((1, 80, 88), dtype=dtype, device=device)
|
||||
elif mode == "normal":
|
||||
mel_input = torch.randn((1, 80, 88), dtype=dtype, device=device)
|
||||
else:
|
||||
raise Exception(f"Mode {mode} if not supported")
|
||||
|
||||
def stft_fn(audio, n_fft, hop_length, win_length, window):
|
||||
spec = torch.stft(
|
||||
audio,
|
||||
n_fft=n_fft,
|
||||
hop_length=hop_length,
|
||||
win_length=win_length,
|
||||
window=window,
|
||||
return_complex=True,
|
||||
)
|
||||
spec = torch.view_as_real(spec)
|
||||
return torch.sqrt(spec.pow(2).sum(-1)), torch.atan2(spec[..., -1], spec[..., 0])
|
||||
|
||||
self.stft = lambda x: stft_fn(
|
||||
audio=x,
|
||||
n_fft=self.filter_length,
|
||||
hop_length=self.hop_length,
|
||||
win_length=self.win_length,
|
||||
window=torch.hann_window(self.win_length, device=device),
|
||||
)
|
||||
self.istft = lambda x, y: torch.istft(
|
||||
torch.complex(x * torch.cos(y), x * torch.sin(y)),
|
||||
n_fft=self.filter_length,
|
||||
hop_length=self.hop_length,
|
||||
win_length=self.win_length,
|
||||
window=torch.hann_window(self.win_length, device=device),
|
||||
)
|
||||
|
||||
with torch.no_grad():
|
||||
bias_audio = vocoder(mel_input).float().squeeze(0)
|
||||
bias_spec, _ = self.stft(bias_audio)
|
||||
|
||||
self.register_buffer("bias_spec", bias_spec[:, :, 0][:, :, None])
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(self, audio, strength=0.0005):
|
||||
audio_spec, audio_angles = self.stft(audio)
|
||||
audio_spec_denoised = audio_spec - self.bias_spec.to(audio.device) * strength
|
||||
audio_spec_denoised = torch.clamp(audio_spec_denoised, 0.0)
|
||||
audio_denoised = self.istft(audio_spec_denoised, audio_angles)
|
||||
return audio_denoised
|
||||
17
vendor/CosyVoice/third_party/Matcha-TTS/matcha/hifigan/env.py
vendored
Normal file
17
vendor/CosyVoice/third_party/Matcha-TTS/matcha/hifigan/env.py
vendored
Normal file
@@ -0,0 +1,17 @@
|
||||
""" from https://github.com/jik876/hifi-gan """
|
||||
|
||||
import os
|
||||
import shutil
|
||||
|
||||
|
||||
class AttrDict(dict):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.__dict__ = self
|
||||
|
||||
|
||||
def build_env(config, config_name, path):
|
||||
t_path = os.path.join(path, config_name)
|
||||
if config != t_path:
|
||||
os.makedirs(path, exist_ok=True)
|
||||
shutil.copyfile(config, os.path.join(path, config_name))
|
||||
217
vendor/CosyVoice/third_party/Matcha-TTS/matcha/hifigan/meldataset.py
vendored
Normal file
217
vendor/CosyVoice/third_party/Matcha-TTS/matcha/hifigan/meldataset.py
vendored
Normal file
@@ -0,0 +1,217 @@
|
||||
""" from https://github.com/jik876/hifi-gan """
|
||||
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.utils.data
|
||||
from librosa.filters import mel as librosa_mel_fn
|
||||
from librosa.util import normalize
|
||||
from scipy.io.wavfile import read
|
||||
|
||||
MAX_WAV_VALUE = 32768.0
|
||||
|
||||
|
||||
def load_wav(full_path):
|
||||
sampling_rate, data = read(full_path)
|
||||
return data, sampling_rate
|
||||
|
||||
|
||||
def dynamic_range_compression(x, C=1, clip_val=1e-5):
|
||||
return np.log(np.clip(x, a_min=clip_val, a_max=None) * C)
|
||||
|
||||
|
||||
def dynamic_range_decompression(x, C=1):
|
||||
return np.exp(x) / C
|
||||
|
||||
|
||||
def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
|
||||
return torch.log(torch.clamp(x, min=clip_val) * C)
|
||||
|
||||
|
||||
def dynamic_range_decompression_torch(x, C=1):
|
||||
return torch.exp(x) / C
|
||||
|
||||
|
||||
def spectral_normalize_torch(magnitudes):
|
||||
output = dynamic_range_compression_torch(magnitudes)
|
||||
return output
|
||||
|
||||
|
||||
def spectral_de_normalize_torch(magnitudes):
|
||||
output = dynamic_range_decompression_torch(magnitudes)
|
||||
return output
|
||||
|
||||
|
||||
mel_basis = {}
|
||||
hann_window = {}
|
||||
|
||||
|
||||
def mel_spectrogram(y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False):
|
||||
if torch.min(y) < -1.0:
|
||||
print("min value is ", torch.min(y))
|
||||
if torch.max(y) > 1.0:
|
||||
print("max value is ", torch.max(y))
|
||||
|
||||
global mel_basis, hann_window # pylint: disable=global-statement
|
||||
if fmax not in mel_basis:
|
||||
mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
|
||||
mel_basis[str(fmax) + "_" + str(y.device)] = torch.from_numpy(mel).float().to(y.device)
|
||||
hann_window[str(y.device)] = torch.hann_window(win_size).to(y.device)
|
||||
|
||||
y = torch.nn.functional.pad(
|
||||
y.unsqueeze(1), (int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)), mode="reflect"
|
||||
)
|
||||
y = y.squeeze(1)
|
||||
|
||||
spec = torch.view_as_real(
|
||||
torch.stft(
|
||||
y,
|
||||
n_fft,
|
||||
hop_length=hop_size,
|
||||
win_length=win_size,
|
||||
window=hann_window[str(y.device)],
|
||||
center=center,
|
||||
pad_mode="reflect",
|
||||
normalized=False,
|
||||
onesided=True,
|
||||
return_complex=True,
|
||||
)
|
||||
)
|
||||
|
||||
spec = torch.sqrt(spec.pow(2).sum(-1) + (1e-9))
|
||||
|
||||
spec = torch.matmul(mel_basis[str(fmax) + "_" + str(y.device)], spec)
|
||||
spec = spectral_normalize_torch(spec)
|
||||
|
||||
return spec
|
||||
|
||||
|
||||
def get_dataset_filelist(a):
|
||||
with open(a.input_training_file, encoding="utf-8") as fi:
|
||||
training_files = [
|
||||
os.path.join(a.input_wavs_dir, x.split("|")[0] + ".wav") for x in fi.read().split("\n") if len(x) > 0
|
||||
]
|
||||
|
||||
with open(a.input_validation_file, encoding="utf-8") as fi:
|
||||
validation_files = [
|
||||
os.path.join(a.input_wavs_dir, x.split("|")[0] + ".wav") for x in fi.read().split("\n") if len(x) > 0
|
||||
]
|
||||
return training_files, validation_files
|
||||
|
||||
|
||||
class MelDataset(torch.utils.data.Dataset):
|
||||
def __init__(
|
||||
self,
|
||||
training_files,
|
||||
segment_size,
|
||||
n_fft,
|
||||
num_mels,
|
||||
hop_size,
|
||||
win_size,
|
||||
sampling_rate,
|
||||
fmin,
|
||||
fmax,
|
||||
split=True,
|
||||
shuffle=True,
|
||||
n_cache_reuse=1,
|
||||
device=None,
|
||||
fmax_loss=None,
|
||||
fine_tuning=False,
|
||||
base_mels_path=None,
|
||||
):
|
||||
self.audio_files = training_files
|
||||
random.seed(1234)
|
||||
if shuffle:
|
||||
random.shuffle(self.audio_files)
|
||||
self.segment_size = segment_size
|
||||
self.sampling_rate = sampling_rate
|
||||
self.split = split
|
||||
self.n_fft = n_fft
|
||||
self.num_mels = num_mels
|
||||
self.hop_size = hop_size
|
||||
self.win_size = win_size
|
||||
self.fmin = fmin
|
||||
self.fmax = fmax
|
||||
self.fmax_loss = fmax_loss
|
||||
self.cached_wav = None
|
||||
self.n_cache_reuse = n_cache_reuse
|
||||
self._cache_ref_count = 0
|
||||
self.device = device
|
||||
self.fine_tuning = fine_tuning
|
||||
self.base_mels_path = base_mels_path
|
||||
|
||||
def __getitem__(self, index):
|
||||
filename = self.audio_files[index]
|
||||
if self._cache_ref_count == 0:
|
||||
audio, sampling_rate = load_wav(filename)
|
||||
audio = audio / MAX_WAV_VALUE
|
||||
if not self.fine_tuning:
|
||||
audio = normalize(audio) * 0.95
|
||||
self.cached_wav = audio
|
||||
if sampling_rate != self.sampling_rate:
|
||||
raise ValueError(f"{sampling_rate} SR doesn't match target {self.sampling_rate} SR")
|
||||
self._cache_ref_count = self.n_cache_reuse
|
||||
else:
|
||||
audio = self.cached_wav
|
||||
self._cache_ref_count -= 1
|
||||
|
||||
audio = torch.FloatTensor(audio)
|
||||
audio = audio.unsqueeze(0)
|
||||
|
||||
if not self.fine_tuning:
|
||||
if self.split:
|
||||
if audio.size(1) >= self.segment_size:
|
||||
max_audio_start = audio.size(1) - self.segment_size
|
||||
audio_start = random.randint(0, max_audio_start)
|
||||
audio = audio[:, audio_start : audio_start + self.segment_size]
|
||||
else:
|
||||
audio = torch.nn.functional.pad(audio, (0, self.segment_size - audio.size(1)), "constant")
|
||||
|
||||
mel = mel_spectrogram(
|
||||
audio,
|
||||
self.n_fft,
|
||||
self.num_mels,
|
||||
self.sampling_rate,
|
||||
self.hop_size,
|
||||
self.win_size,
|
||||
self.fmin,
|
||||
self.fmax,
|
||||
center=False,
|
||||
)
|
||||
else:
|
||||
mel = np.load(os.path.join(self.base_mels_path, os.path.splitext(os.path.split(filename)[-1])[0] + ".npy"))
|
||||
mel = torch.from_numpy(mel)
|
||||
|
||||
if len(mel.shape) < 3:
|
||||
mel = mel.unsqueeze(0)
|
||||
|
||||
if self.split:
|
||||
frames_per_seg = math.ceil(self.segment_size / self.hop_size)
|
||||
|
||||
if audio.size(1) >= self.segment_size:
|
||||
mel_start = random.randint(0, mel.size(2) - frames_per_seg - 1)
|
||||
mel = mel[:, :, mel_start : mel_start + frames_per_seg]
|
||||
audio = audio[:, mel_start * self.hop_size : (mel_start + frames_per_seg) * self.hop_size]
|
||||
else:
|
||||
mel = torch.nn.functional.pad(mel, (0, frames_per_seg - mel.size(2)), "constant")
|
||||
audio = torch.nn.functional.pad(audio, (0, self.segment_size - audio.size(1)), "constant")
|
||||
|
||||
mel_loss = mel_spectrogram(
|
||||
audio,
|
||||
self.n_fft,
|
||||
self.num_mels,
|
||||
self.sampling_rate,
|
||||
self.hop_size,
|
||||
self.win_size,
|
||||
self.fmin,
|
||||
self.fmax_loss,
|
||||
center=False,
|
||||
)
|
||||
|
||||
return (mel.squeeze(), audio.squeeze(0), filename, mel_loss.squeeze())
|
||||
|
||||
def __len__(self):
|
||||
return len(self.audio_files)
|
||||
368
vendor/CosyVoice/third_party/Matcha-TTS/matcha/hifigan/models.py
vendored
Normal file
368
vendor/CosyVoice/third_party/Matcha-TTS/matcha/hifigan/models.py
vendored
Normal file
@@ -0,0 +1,368 @@
|
||||
""" from https://github.com/jik876/hifi-gan """
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch.nn import AvgPool1d, Conv1d, Conv2d, ConvTranspose1d
|
||||
from torch.nn.utils import remove_weight_norm, spectral_norm, weight_norm
|
||||
|
||||
from .xutils import get_padding, init_weights
|
||||
|
||||
LRELU_SLOPE = 0.1
|
||||
|
||||
|
||||
class ResBlock1(torch.nn.Module):
|
||||
def __init__(self, h, channels, kernel_size=3, dilation=(1, 3, 5)):
|
||||
super().__init__()
|
||||
self.h = h
|
||||
self.convs1 = nn.ModuleList(
|
||||
[
|
||||
weight_norm(
|
||||
Conv1d(
|
||||
channels,
|
||||
channels,
|
||||
kernel_size,
|
||||
1,
|
||||
dilation=dilation[0],
|
||||
padding=get_padding(kernel_size, dilation[0]),
|
||||
)
|
||||
),
|
||||
weight_norm(
|
||||
Conv1d(
|
||||
channels,
|
||||
channels,
|
||||
kernel_size,
|
||||
1,
|
||||
dilation=dilation[1],
|
||||
padding=get_padding(kernel_size, dilation[1]),
|
||||
)
|
||||
),
|
||||
weight_norm(
|
||||
Conv1d(
|
||||
channels,
|
||||
channels,
|
||||
kernel_size,
|
||||
1,
|
||||
dilation=dilation[2],
|
||||
padding=get_padding(kernel_size, dilation[2]),
|
||||
)
|
||||
),
|
||||
]
|
||||
)
|
||||
self.convs1.apply(init_weights)
|
||||
|
||||
self.convs2 = nn.ModuleList(
|
||||
[
|
||||
weight_norm(
|
||||
Conv1d(
|
||||
channels,
|
||||
channels,
|
||||
kernel_size,
|
||||
1,
|
||||
dilation=1,
|
||||
padding=get_padding(kernel_size, 1),
|
||||
)
|
||||
),
|
||||
weight_norm(
|
||||
Conv1d(
|
||||
channels,
|
||||
channels,
|
||||
kernel_size,
|
||||
1,
|
||||
dilation=1,
|
||||
padding=get_padding(kernel_size, 1),
|
||||
)
|
||||
),
|
||||
weight_norm(
|
||||
Conv1d(
|
||||
channels,
|
||||
channels,
|
||||
kernel_size,
|
||||
1,
|
||||
dilation=1,
|
||||
padding=get_padding(kernel_size, 1),
|
||||
)
|
||||
),
|
||||
]
|
||||
)
|
||||
self.convs2.apply(init_weights)
|
||||
|
||||
def forward(self, x):
|
||||
for c1, c2 in zip(self.convs1, self.convs2):
|
||||
xt = F.leaky_relu(x, LRELU_SLOPE)
|
||||
xt = c1(xt)
|
||||
xt = F.leaky_relu(xt, LRELU_SLOPE)
|
||||
xt = c2(xt)
|
||||
x = xt + x
|
||||
return x
|
||||
|
||||
def remove_weight_norm(self):
|
||||
for l in self.convs1:
|
||||
remove_weight_norm(l)
|
||||
for l in self.convs2:
|
||||
remove_weight_norm(l)
|
||||
|
||||
|
||||
class ResBlock2(torch.nn.Module):
|
||||
def __init__(self, h, channels, kernel_size=3, dilation=(1, 3)):
|
||||
super().__init__()
|
||||
self.h = h
|
||||
self.convs = nn.ModuleList(
|
||||
[
|
||||
weight_norm(
|
||||
Conv1d(
|
||||
channels,
|
||||
channels,
|
||||
kernel_size,
|
||||
1,
|
||||
dilation=dilation[0],
|
||||
padding=get_padding(kernel_size, dilation[0]),
|
||||
)
|
||||
),
|
||||
weight_norm(
|
||||
Conv1d(
|
||||
channels,
|
||||
channels,
|
||||
kernel_size,
|
||||
1,
|
||||
dilation=dilation[1],
|
||||
padding=get_padding(kernel_size, dilation[1]),
|
||||
)
|
||||
),
|
||||
]
|
||||
)
|
||||
self.convs.apply(init_weights)
|
||||
|
||||
def forward(self, x):
|
||||
for c in self.convs:
|
||||
xt = F.leaky_relu(x, LRELU_SLOPE)
|
||||
xt = c(xt)
|
||||
x = xt + x
|
||||
return x
|
||||
|
||||
def remove_weight_norm(self):
|
||||
for l in self.convs:
|
||||
remove_weight_norm(l)
|
||||
|
||||
|
||||
class Generator(torch.nn.Module):
|
||||
def __init__(self, h):
|
||||
super().__init__()
|
||||
self.h = h
|
||||
self.num_kernels = len(h.resblock_kernel_sizes)
|
||||
self.num_upsamples = len(h.upsample_rates)
|
||||
self.conv_pre = weight_norm(Conv1d(80, h.upsample_initial_channel, 7, 1, padding=3))
|
||||
resblock = ResBlock1 if h.resblock == "1" else ResBlock2
|
||||
|
||||
self.ups = nn.ModuleList()
|
||||
for i, (u, k) in enumerate(zip(h.upsample_rates, h.upsample_kernel_sizes)):
|
||||
self.ups.append(
|
||||
weight_norm(
|
||||
ConvTranspose1d(
|
||||
h.upsample_initial_channel // (2**i),
|
||||
h.upsample_initial_channel // (2 ** (i + 1)),
|
||||
k,
|
||||
u,
|
||||
padding=(k - u) // 2,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
self.resblocks = nn.ModuleList()
|
||||
for i in range(len(self.ups)):
|
||||
ch = h.upsample_initial_channel // (2 ** (i + 1))
|
||||
for _, (k, d) in enumerate(zip(h.resblock_kernel_sizes, h.resblock_dilation_sizes)):
|
||||
self.resblocks.append(resblock(h, ch, k, d))
|
||||
|
||||
self.conv_post = weight_norm(Conv1d(ch, 1, 7, 1, padding=3))
|
||||
self.ups.apply(init_weights)
|
||||
self.conv_post.apply(init_weights)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.conv_pre(x)
|
||||
for i in range(self.num_upsamples):
|
||||
x = F.leaky_relu(x, LRELU_SLOPE)
|
||||
x = self.ups[i](x)
|
||||
xs = None
|
||||
for j in range(self.num_kernels):
|
||||
if xs is None:
|
||||
xs = self.resblocks[i * self.num_kernels + j](x)
|
||||
else:
|
||||
xs += self.resblocks[i * self.num_kernels + j](x)
|
||||
x = xs / self.num_kernels
|
||||
x = F.leaky_relu(x)
|
||||
x = self.conv_post(x)
|
||||
x = torch.tanh(x)
|
||||
|
||||
return x
|
||||
|
||||
def remove_weight_norm(self):
|
||||
print("Removing weight norm...")
|
||||
for l in self.ups:
|
||||
remove_weight_norm(l)
|
||||
for l in self.resblocks:
|
||||
l.remove_weight_norm()
|
||||
remove_weight_norm(self.conv_pre)
|
||||
remove_weight_norm(self.conv_post)
|
||||
|
||||
|
||||
class DiscriminatorP(torch.nn.Module):
|
||||
def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
|
||||
super().__init__()
|
||||
self.period = period
|
||||
norm_f = weight_norm if use_spectral_norm is False else spectral_norm
|
||||
self.convs = nn.ModuleList(
|
||||
[
|
||||
norm_f(Conv2d(1, 32, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))),
|
||||
norm_f(Conv2d(32, 128, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))),
|
||||
norm_f(Conv2d(128, 512, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))),
|
||||
norm_f(Conv2d(512, 1024, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))),
|
||||
norm_f(Conv2d(1024, 1024, (kernel_size, 1), 1, padding=(2, 0))),
|
||||
]
|
||||
)
|
||||
self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
|
||||
|
||||
def forward(self, x):
|
||||
fmap = []
|
||||
|
||||
# 1d to 2d
|
||||
b, c, t = x.shape
|
||||
if t % self.period != 0: # pad first
|
||||
n_pad = self.period - (t % self.period)
|
||||
x = F.pad(x, (0, n_pad), "reflect")
|
||||
t = t + n_pad
|
||||
x = x.view(b, c, t // self.period, self.period)
|
||||
|
||||
for l in self.convs:
|
||||
x = l(x)
|
||||
x = F.leaky_relu(x, LRELU_SLOPE)
|
||||
fmap.append(x)
|
||||
x = self.conv_post(x)
|
||||
fmap.append(x)
|
||||
x = torch.flatten(x, 1, -1)
|
||||
|
||||
return x, fmap
|
||||
|
||||
|
||||
class MultiPeriodDiscriminator(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.discriminators = nn.ModuleList(
|
||||
[
|
||||
DiscriminatorP(2),
|
||||
DiscriminatorP(3),
|
||||
DiscriminatorP(5),
|
||||
DiscriminatorP(7),
|
||||
DiscriminatorP(11),
|
||||
]
|
||||
)
|
||||
|
||||
def forward(self, y, y_hat):
|
||||
y_d_rs = []
|
||||
y_d_gs = []
|
||||
fmap_rs = []
|
||||
fmap_gs = []
|
||||
for _, d in enumerate(self.discriminators):
|
||||
y_d_r, fmap_r = d(y)
|
||||
y_d_g, fmap_g = d(y_hat)
|
||||
y_d_rs.append(y_d_r)
|
||||
fmap_rs.append(fmap_r)
|
||||
y_d_gs.append(y_d_g)
|
||||
fmap_gs.append(fmap_g)
|
||||
|
||||
return y_d_rs, y_d_gs, fmap_rs, fmap_gs
|
||||
|
||||
|
||||
class DiscriminatorS(torch.nn.Module):
|
||||
def __init__(self, use_spectral_norm=False):
|
||||
super().__init__()
|
||||
norm_f = weight_norm if use_spectral_norm is False else spectral_norm
|
||||
self.convs = nn.ModuleList(
|
||||
[
|
||||
norm_f(Conv1d(1, 128, 15, 1, padding=7)),
|
||||
norm_f(Conv1d(128, 128, 41, 2, groups=4, padding=20)),
|
||||
norm_f(Conv1d(128, 256, 41, 2, groups=16, padding=20)),
|
||||
norm_f(Conv1d(256, 512, 41, 4, groups=16, padding=20)),
|
||||
norm_f(Conv1d(512, 1024, 41, 4, groups=16, padding=20)),
|
||||
norm_f(Conv1d(1024, 1024, 41, 1, groups=16, padding=20)),
|
||||
norm_f(Conv1d(1024, 1024, 5, 1, padding=2)),
|
||||
]
|
||||
)
|
||||
self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1))
|
||||
|
||||
def forward(self, x):
|
||||
fmap = []
|
||||
for l in self.convs:
|
||||
x = l(x)
|
||||
x = F.leaky_relu(x, LRELU_SLOPE)
|
||||
fmap.append(x)
|
||||
x = self.conv_post(x)
|
||||
fmap.append(x)
|
||||
x = torch.flatten(x, 1, -1)
|
||||
|
||||
return x, fmap
|
||||
|
||||
|
||||
class MultiScaleDiscriminator(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.discriminators = nn.ModuleList(
|
||||
[
|
||||
DiscriminatorS(use_spectral_norm=True),
|
||||
DiscriminatorS(),
|
||||
DiscriminatorS(),
|
||||
]
|
||||
)
|
||||
self.meanpools = nn.ModuleList([AvgPool1d(4, 2, padding=2), AvgPool1d(4, 2, padding=2)])
|
||||
|
||||
def forward(self, y, y_hat):
|
||||
y_d_rs = []
|
||||
y_d_gs = []
|
||||
fmap_rs = []
|
||||
fmap_gs = []
|
||||
for i, d in enumerate(self.discriminators):
|
||||
if i != 0:
|
||||
y = self.meanpools[i - 1](y)
|
||||
y_hat = self.meanpools[i - 1](y_hat)
|
||||
y_d_r, fmap_r = d(y)
|
||||
y_d_g, fmap_g = d(y_hat)
|
||||
y_d_rs.append(y_d_r)
|
||||
fmap_rs.append(fmap_r)
|
||||
y_d_gs.append(y_d_g)
|
||||
fmap_gs.append(fmap_g)
|
||||
|
||||
return y_d_rs, y_d_gs, fmap_rs, fmap_gs
|
||||
|
||||
|
||||
def feature_loss(fmap_r, fmap_g):
|
||||
loss = 0
|
||||
for dr, dg in zip(fmap_r, fmap_g):
|
||||
for rl, gl in zip(dr, dg):
|
||||
loss += torch.mean(torch.abs(rl - gl))
|
||||
|
||||
return loss * 2
|
||||
|
||||
|
||||
def discriminator_loss(disc_real_outputs, disc_generated_outputs):
|
||||
loss = 0
|
||||
r_losses = []
|
||||
g_losses = []
|
||||
for dr, dg in zip(disc_real_outputs, disc_generated_outputs):
|
||||
r_loss = torch.mean((1 - dr) ** 2)
|
||||
g_loss = torch.mean(dg**2)
|
||||
loss += r_loss + g_loss
|
||||
r_losses.append(r_loss.item())
|
||||
g_losses.append(g_loss.item())
|
||||
|
||||
return loss, r_losses, g_losses
|
||||
|
||||
|
||||
def generator_loss(disc_outputs):
|
||||
loss = 0
|
||||
gen_losses = []
|
||||
for dg in disc_outputs:
|
||||
l = torch.mean((1 - dg) ** 2)
|
||||
gen_losses.append(l)
|
||||
loss += l
|
||||
|
||||
return loss, gen_losses
|
||||
60
vendor/CosyVoice/third_party/Matcha-TTS/matcha/hifigan/xutils.py
vendored
Normal file
60
vendor/CosyVoice/third_party/Matcha-TTS/matcha/hifigan/xutils.py
vendored
Normal file
@@ -0,0 +1,60 @@
|
||||
""" from https://github.com/jik876/hifi-gan """
|
||||
|
||||
import glob
|
||||
import os
|
||||
|
||||
import matplotlib
|
||||
import torch
|
||||
from torch.nn.utils import weight_norm
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pylab as plt
|
||||
|
||||
|
||||
def plot_spectrogram(spectrogram):
|
||||
fig, ax = plt.subplots(figsize=(10, 2))
|
||||
im = ax.imshow(spectrogram, aspect="auto", origin="lower", interpolation="none")
|
||||
plt.colorbar(im, ax=ax)
|
||||
|
||||
fig.canvas.draw()
|
||||
plt.close()
|
||||
|
||||
return fig
|
||||
|
||||
|
||||
def init_weights(m, mean=0.0, std=0.01):
|
||||
classname = m.__class__.__name__
|
||||
if classname.find("Conv") != -1:
|
||||
m.weight.data.normal_(mean, std)
|
||||
|
||||
|
||||
def apply_weight_norm(m):
|
||||
classname = m.__class__.__name__
|
||||
if classname.find("Conv") != -1:
|
||||
weight_norm(m)
|
||||
|
||||
|
||||
def get_padding(kernel_size, dilation=1):
|
||||
return int((kernel_size * dilation - dilation) / 2)
|
||||
|
||||
|
||||
def load_checkpoint(filepath, device):
|
||||
assert os.path.isfile(filepath)
|
||||
print(f"Loading '{filepath}'")
|
||||
checkpoint_dict = torch.load(filepath, map_location=device)
|
||||
print("Complete.")
|
||||
return checkpoint_dict
|
||||
|
||||
|
||||
def save_checkpoint(filepath, obj):
|
||||
print(f"Saving checkpoint to {filepath}")
|
||||
torch.save(obj, filepath)
|
||||
print("Complete.")
|
||||
|
||||
|
||||
def scan_checkpoint(cp_dir, prefix):
|
||||
pattern = os.path.join(cp_dir, prefix + "????????")
|
||||
cp_list = glob.glob(pattern)
|
||||
if len(cp_list) == 0:
|
||||
return None
|
||||
return sorted(cp_list)[-1]
|
||||
0
vendor/CosyVoice/third_party/Matcha-TTS/matcha/onnx/__init__.py
vendored
Normal file
0
vendor/CosyVoice/third_party/Matcha-TTS/matcha/onnx/__init__.py
vendored
Normal file
181
vendor/CosyVoice/third_party/Matcha-TTS/matcha/onnx/export.py
vendored
Normal file
181
vendor/CosyVoice/third_party/Matcha-TTS/matcha/onnx/export.py
vendored
Normal file
@@ -0,0 +1,181 @@
|
||||
import argparse
|
||||
import random
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from lightning import LightningModule
|
||||
|
||||
from matcha.cli import VOCODER_URLS, load_matcha, load_vocoder
|
||||
|
||||
DEFAULT_OPSET = 15
|
||||
|
||||
SEED = 1234
|
||||
random.seed(SEED)
|
||||
np.random.seed(SEED)
|
||||
torch.manual_seed(SEED)
|
||||
torch.cuda.manual_seed(SEED)
|
||||
torch.backends.cudnn.deterministic = True
|
||||
torch.backends.cudnn.benchmark = False
|
||||
|
||||
|
||||
class MatchaWithVocoder(LightningModule):
|
||||
def __init__(self, matcha, vocoder):
|
||||
super().__init__()
|
||||
self.matcha = matcha
|
||||
self.vocoder = vocoder
|
||||
|
||||
def forward(self, x, x_lengths, scales, spks=None):
|
||||
mel, mel_lengths = self.matcha(x, x_lengths, scales, spks)
|
||||
wavs = self.vocoder(mel).clamp(-1, 1)
|
||||
lengths = mel_lengths * 256
|
||||
return wavs.squeeze(1), lengths
|
||||
|
||||
|
||||
def get_exportable_module(matcha, vocoder, n_timesteps):
|
||||
"""
|
||||
Return an appropriate `LighteningModule` and output-node names
|
||||
based on whether the vocoder is embedded in the final graph
|
||||
"""
|
||||
|
||||
def onnx_forward_func(x, x_lengths, scales, spks=None):
|
||||
"""
|
||||
Custom forward function for accepting
|
||||
scaler parameters as tensors
|
||||
"""
|
||||
# Extract scaler parameters from tensors
|
||||
temperature = scales[0]
|
||||
length_scale = scales[1]
|
||||
output = matcha.synthesise(x, x_lengths, n_timesteps, temperature, spks, length_scale)
|
||||
return output["mel"], output["mel_lengths"]
|
||||
|
||||
# Monkey-patch Matcha's forward function
|
||||
matcha.forward = onnx_forward_func
|
||||
|
||||
if vocoder is None:
|
||||
model, output_names = matcha, ["mel", "mel_lengths"]
|
||||
else:
|
||||
model = MatchaWithVocoder(matcha, vocoder)
|
||||
output_names = ["wav", "wav_lengths"]
|
||||
return model, output_names
|
||||
|
||||
|
||||
def get_inputs(is_multi_speaker):
|
||||
"""
|
||||
Create dummy inputs for tracing
|
||||
"""
|
||||
dummy_input_length = 50
|
||||
x = torch.randint(low=0, high=20, size=(1, dummy_input_length), dtype=torch.long)
|
||||
x_lengths = torch.LongTensor([dummy_input_length])
|
||||
|
||||
# Scales
|
||||
temperature = 0.667
|
||||
length_scale = 1.0
|
||||
scales = torch.Tensor([temperature, length_scale])
|
||||
|
||||
model_inputs = [x, x_lengths, scales]
|
||||
input_names = [
|
||||
"x",
|
||||
"x_lengths",
|
||||
"scales",
|
||||
]
|
||||
|
||||
if is_multi_speaker:
|
||||
spks = torch.LongTensor([1])
|
||||
model_inputs.append(spks)
|
||||
input_names.append("spks")
|
||||
|
||||
return tuple(model_inputs), input_names
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Export 🍵 Matcha-TTS to ONNX")
|
||||
|
||||
parser.add_argument(
|
||||
"checkpoint_path",
|
||||
type=str,
|
||||
help="Path to the model checkpoint",
|
||||
)
|
||||
parser.add_argument("output", type=str, help="Path to output `.onnx` file")
|
||||
parser.add_argument(
|
||||
"--n-timesteps", type=int, default=5, help="Number of steps to use for reverse diffusion in decoder (default 5)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--vocoder-name",
|
||||
type=str,
|
||||
choices=list(VOCODER_URLS.keys()),
|
||||
default=None,
|
||||
help="Name of the vocoder to embed in the ONNX graph",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--vocoder-checkpoint-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Vocoder checkpoint to embed in the ONNX graph for an `e2e` like experience",
|
||||
)
|
||||
parser.add_argument("--opset", type=int, default=DEFAULT_OPSET, help="ONNX opset version to use (default 15")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"[🍵] Loading Matcha checkpoint from {args.checkpoint_path}")
|
||||
print(f"Setting n_timesteps to {args.n_timesteps}")
|
||||
|
||||
checkpoint_path = Path(args.checkpoint_path)
|
||||
matcha = load_matcha(checkpoint_path.stem, checkpoint_path, "cpu")
|
||||
|
||||
if args.vocoder_name or args.vocoder_checkpoint_path:
|
||||
assert (
|
||||
args.vocoder_name and args.vocoder_checkpoint_path
|
||||
), "Both vocoder_name and vocoder-checkpoint are required when embedding the vocoder in the ONNX graph."
|
||||
vocoder, _ = load_vocoder(args.vocoder_name, args.vocoder_checkpoint_path, "cpu")
|
||||
else:
|
||||
vocoder = None
|
||||
|
||||
is_multi_speaker = matcha.n_spks > 1
|
||||
|
||||
dummy_input, input_names = get_inputs(is_multi_speaker)
|
||||
model, output_names = get_exportable_module(matcha, vocoder, args.n_timesteps)
|
||||
|
||||
# Set dynamic shape for inputs/outputs
|
||||
dynamic_axes = {
|
||||
"x": {0: "batch_size", 1: "time"},
|
||||
"x_lengths": {0: "batch_size"},
|
||||
}
|
||||
|
||||
if vocoder is None:
|
||||
dynamic_axes.update(
|
||||
{
|
||||
"mel": {0: "batch_size", 2: "time"},
|
||||
"mel_lengths": {0: "batch_size"},
|
||||
}
|
||||
)
|
||||
else:
|
||||
print("Embedding the vocoder in the ONNX graph")
|
||||
dynamic_axes.update(
|
||||
{
|
||||
"wav": {0: "batch_size", 1: "time"},
|
||||
"wav_lengths": {0: "batch_size"},
|
||||
}
|
||||
)
|
||||
|
||||
if is_multi_speaker:
|
||||
dynamic_axes["spks"] = {0: "batch_size"}
|
||||
|
||||
# Create the output directory (if not exists)
|
||||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
model.to_onnx(
|
||||
args.output,
|
||||
dummy_input,
|
||||
input_names=input_names,
|
||||
output_names=output_names,
|
||||
dynamic_axes=dynamic_axes,
|
||||
opset_version=args.opset,
|
||||
export_params=True,
|
||||
do_constant_folding=True,
|
||||
)
|
||||
print(f"[🍵] ONNX model exported to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
168
vendor/CosyVoice/third_party/Matcha-TTS/matcha/onnx/infer.py
vendored
Normal file
168
vendor/CosyVoice/third_party/Matcha-TTS/matcha/onnx/infer.py
vendored
Normal file
@@ -0,0 +1,168 @@
|
||||
import argparse
|
||||
import os
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from time import perf_counter
|
||||
|
||||
import numpy as np
|
||||
import onnxruntime as ort
|
||||
import soundfile as sf
|
||||
import torch
|
||||
|
||||
from matcha.cli import plot_spectrogram_to_numpy, process_text
|
||||
|
||||
|
||||
def validate_args(args):
|
||||
assert (
|
||||
args.text or args.file
|
||||
), "Either text or file must be provided Matcha-T(ea)TTS need sometext to whisk the waveforms."
|
||||
assert args.temperature >= 0, "Sampling temperature cannot be negative"
|
||||
assert args.speaking_rate >= 0, "Speaking rate must be greater than 0"
|
||||
return args
|
||||
|
||||
|
||||
def write_wavs(model, inputs, output_dir, external_vocoder=None):
|
||||
if external_vocoder is None:
|
||||
print("The provided model has the vocoder embedded in the graph.\nGenerating waveform directly")
|
||||
t0 = perf_counter()
|
||||
wavs, wav_lengths = model.run(None, inputs)
|
||||
infer_secs = perf_counter() - t0
|
||||
mel_infer_secs = vocoder_infer_secs = None
|
||||
else:
|
||||
print("[🍵] Generating mel using Matcha")
|
||||
mel_t0 = perf_counter()
|
||||
mels, mel_lengths = model.run(None, inputs)
|
||||
mel_infer_secs = perf_counter() - mel_t0
|
||||
print("Generating waveform from mel using external vocoder")
|
||||
vocoder_inputs = {external_vocoder.get_inputs()[0].name: mels}
|
||||
vocoder_t0 = perf_counter()
|
||||
wavs = external_vocoder.run(None, vocoder_inputs)[0]
|
||||
vocoder_infer_secs = perf_counter() - vocoder_t0
|
||||
wavs = wavs.squeeze(1)
|
||||
wav_lengths = mel_lengths * 256
|
||||
infer_secs = mel_infer_secs + vocoder_infer_secs
|
||||
|
||||
output_dir = Path(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
for i, (wav, wav_length) in enumerate(zip(wavs, wav_lengths)):
|
||||
output_filename = output_dir.joinpath(f"output_{i + 1}.wav")
|
||||
audio = wav[:wav_length]
|
||||
print(f"Writing audio to {output_filename}")
|
||||
sf.write(output_filename, audio, 22050, "PCM_24")
|
||||
|
||||
wav_secs = wav_lengths.sum() / 22050
|
||||
print(f"Inference seconds: {infer_secs}")
|
||||
print(f"Generated wav seconds: {wav_secs}")
|
||||
rtf = infer_secs / wav_secs
|
||||
if mel_infer_secs is not None:
|
||||
mel_rtf = mel_infer_secs / wav_secs
|
||||
print(f"Matcha RTF: {mel_rtf}")
|
||||
if vocoder_infer_secs is not None:
|
||||
vocoder_rtf = vocoder_infer_secs / wav_secs
|
||||
print(f"Vocoder RTF: {vocoder_rtf}")
|
||||
print(f"Overall RTF: {rtf}")
|
||||
|
||||
|
||||
def write_mels(model, inputs, output_dir):
|
||||
t0 = perf_counter()
|
||||
mels, mel_lengths = model.run(None, inputs)
|
||||
infer_secs = perf_counter() - t0
|
||||
|
||||
output_dir = Path(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
for i, mel in enumerate(mels):
|
||||
output_stem = output_dir.joinpath(f"output_{i + 1}")
|
||||
plot_spectrogram_to_numpy(mel.squeeze(), output_stem.with_suffix(".png"))
|
||||
np.save(output_stem.with_suffix(".numpy"), mel)
|
||||
|
||||
wav_secs = (mel_lengths * 256).sum() / 22050
|
||||
print(f"Inference seconds: {infer_secs}")
|
||||
print(f"Generated wav seconds: {wav_secs}")
|
||||
rtf = infer_secs / wav_secs
|
||||
print(f"RTF: {rtf}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description=" 🍵 Matcha-TTS: A fast TTS architecture with conditional flow matching"
|
||||
)
|
||||
parser.add_argument(
|
||||
"model",
|
||||
type=str,
|
||||
help="ONNX model to use",
|
||||
)
|
||||
parser.add_argument("--vocoder", type=str, default=None, help="Vocoder to use (defaults to None)")
|
||||
parser.add_argument("--text", type=str, default=None, help="Text to synthesize")
|
||||
parser.add_argument("--file", type=str, default=None, help="Text file to synthesize")
|
||||
parser.add_argument("--spk", type=int, default=None, help="Speaker ID")
|
||||
parser.add_argument(
|
||||
"--temperature",
|
||||
type=float,
|
||||
default=0.667,
|
||||
help="Variance of the x0 noise (default: 0.667)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--speaking-rate",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help="change the speaking rate, a higher value means slower speaking rate (default: 1.0)",
|
||||
)
|
||||
parser.add_argument("--gpu", action="store_true", help="Use CPU for inference (default: use GPU if available)")
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
type=str,
|
||||
default=os.getcwd(),
|
||||
help="Output folder to save results (default: current dir)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
args = validate_args(args)
|
||||
|
||||
if args.gpu:
|
||||
providers = ["GPUExecutionProvider"]
|
||||
else:
|
||||
providers = ["CPUExecutionProvider"]
|
||||
model = ort.InferenceSession(args.model, providers=providers)
|
||||
|
||||
model_inputs = model.get_inputs()
|
||||
model_outputs = list(model.get_outputs())
|
||||
|
||||
if args.text:
|
||||
text_lines = args.text.splitlines()
|
||||
else:
|
||||
with open(args.file, encoding="utf-8") as file:
|
||||
text_lines = file.read().splitlines()
|
||||
|
||||
processed_lines = [process_text(0, line, "cpu") for line in text_lines]
|
||||
x = [line["x"].squeeze() for line in processed_lines]
|
||||
# Pad
|
||||
x = torch.nn.utils.rnn.pad_sequence(x, batch_first=True)
|
||||
x = x.detach().cpu().numpy()
|
||||
x_lengths = np.array([line["x_lengths"].item() for line in processed_lines], dtype=np.int64)
|
||||
inputs = {
|
||||
"x": x,
|
||||
"x_lengths": x_lengths,
|
||||
"scales": np.array([args.temperature, args.speaking_rate], dtype=np.float32),
|
||||
}
|
||||
is_multi_speaker = len(model_inputs) == 4
|
||||
if is_multi_speaker:
|
||||
if args.spk is None:
|
||||
args.spk = 0
|
||||
warn = "[!] Speaker ID not provided! Using speaker ID 0"
|
||||
warnings.warn(warn, UserWarning)
|
||||
inputs["spks"] = np.repeat(args.spk, x.shape[0]).astype(np.int64)
|
||||
|
||||
has_vocoder_embedded = model_outputs[0].name == "wav"
|
||||
if has_vocoder_embedded:
|
||||
write_wavs(model, inputs, args.output_dir)
|
||||
elif args.vocoder:
|
||||
external_vocoder = ort.InferenceSession(args.vocoder, providers=providers)
|
||||
write_wavs(model, inputs, args.output_dir, external_vocoder=external_vocoder)
|
||||
else:
|
||||
warn = "[!] A vocoder is not embedded in the graph nor an external vocoder is provided. The mel output will be written as numpy arrays to `*.npy` files in the output directory"
|
||||
warnings.warn(warn, UserWarning)
|
||||
write_mels(model, inputs, args.output_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
53
vendor/CosyVoice/third_party/Matcha-TTS/matcha/text/__init__.py
vendored
Normal file
53
vendor/CosyVoice/third_party/Matcha-TTS/matcha/text/__init__.py
vendored
Normal file
@@ -0,0 +1,53 @@
|
||||
""" from https://github.com/keithito/tacotron """
|
||||
from matcha.text import cleaners
|
||||
from matcha.text.symbols import symbols
|
||||
|
||||
# Mappings from symbol to numeric ID and vice versa:
|
||||
_symbol_to_id = {s: i for i, s in enumerate(symbols)}
|
||||
_id_to_symbol = {i: s for i, s in enumerate(symbols)} # pylint: disable=unnecessary-comprehension
|
||||
|
||||
|
||||
def text_to_sequence(text, cleaner_names):
|
||||
"""Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
|
||||
Args:
|
||||
text: string to convert to a sequence
|
||||
cleaner_names: names of the cleaner functions to run the text through
|
||||
Returns:
|
||||
List of integers corresponding to the symbols in the text
|
||||
"""
|
||||
sequence = []
|
||||
|
||||
clean_text = _clean_text(text, cleaner_names)
|
||||
for symbol in clean_text:
|
||||
symbol_id = _symbol_to_id[symbol]
|
||||
sequence += [symbol_id]
|
||||
return sequence
|
||||
|
||||
|
||||
def cleaned_text_to_sequence(cleaned_text):
|
||||
"""Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
|
||||
Args:
|
||||
text: string to convert to a sequence
|
||||
Returns:
|
||||
List of integers corresponding to the symbols in the text
|
||||
"""
|
||||
sequence = [_symbol_to_id[symbol] for symbol in cleaned_text]
|
||||
return sequence
|
||||
|
||||
|
||||
def sequence_to_text(sequence):
|
||||
"""Converts a sequence of IDs back to a string"""
|
||||
result = ""
|
||||
for symbol_id in sequence:
|
||||
s = _id_to_symbol[symbol_id]
|
||||
result += s
|
||||
return result
|
||||
|
||||
|
||||
def _clean_text(text, cleaner_names):
|
||||
for name in cleaner_names:
|
||||
cleaner = getattr(cleaners, name)
|
||||
if not cleaner:
|
||||
raise Exception("Unknown cleaner: %s" % name)
|
||||
text = cleaner(text)
|
||||
return text
|
||||
116
vendor/CosyVoice/third_party/Matcha-TTS/matcha/text/cleaners.py
vendored
Normal file
116
vendor/CosyVoice/third_party/Matcha-TTS/matcha/text/cleaners.py
vendored
Normal file
@@ -0,0 +1,116 @@
|
||||
""" from https://github.com/keithito/tacotron
|
||||
|
||||
Cleaners are transformations that run over the input text at both training and eval time.
|
||||
|
||||
Cleaners can be selected by passing a comma-delimited list of cleaner names as the "cleaners"
|
||||
hyperparameter. Some cleaners are English-specific. You'll typically want to use:
|
||||
1. "english_cleaners" for English text
|
||||
2. "transliteration_cleaners" for non-English text that can be transliterated to ASCII using
|
||||
the Unidecode library (https://pypi.python.org/pypi/Unidecode)
|
||||
3. "basic_cleaners" if you do not want to transliterate (in this case, you should also update
|
||||
the symbols in symbols.py to match your data).
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
import phonemizer
|
||||
import piper_phonemize
|
||||
from unidecode import unidecode
|
||||
|
||||
# To avoid excessive logging we set the log level of the phonemizer package to Critical
|
||||
critical_logger = logging.getLogger("phonemizer")
|
||||
critical_logger.setLevel(logging.CRITICAL)
|
||||
|
||||
# Intializing the phonemizer globally significantly reduces the speed
|
||||
# now the phonemizer is not initialising at every call
|
||||
# Might be less flexible, but it is much-much faster
|
||||
global_phonemizer = phonemizer.backend.EspeakBackend(
|
||||
language="en-us",
|
||||
preserve_punctuation=True,
|
||||
with_stress=True,
|
||||
language_switch="remove-flags",
|
||||
logger=critical_logger,
|
||||
)
|
||||
|
||||
|
||||
# Regular expression matching whitespace:
|
||||
_whitespace_re = re.compile(r"\s+")
|
||||
|
||||
# List of (regular expression, replacement) pairs for abbreviations:
|
||||
_abbreviations = [
|
||||
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
||||
for x in [
|
||||
("mrs", "misess"),
|
||||
("mr", "mister"),
|
||||
("dr", "doctor"),
|
||||
("st", "saint"),
|
||||
("co", "company"),
|
||||
("jr", "junior"),
|
||||
("maj", "major"),
|
||||
("gen", "general"),
|
||||
("drs", "doctors"),
|
||||
("rev", "reverend"),
|
||||
("lt", "lieutenant"),
|
||||
("hon", "honorable"),
|
||||
("sgt", "sergeant"),
|
||||
("capt", "captain"),
|
||||
("esq", "esquire"),
|
||||
("ltd", "limited"),
|
||||
("col", "colonel"),
|
||||
("ft", "fort"),
|
||||
]
|
||||
]
|
||||
|
||||
|
||||
def expand_abbreviations(text):
|
||||
for regex, replacement in _abbreviations:
|
||||
text = re.sub(regex, replacement, text)
|
||||
return text
|
||||
|
||||
|
||||
def lowercase(text):
|
||||
return text.lower()
|
||||
|
||||
|
||||
def collapse_whitespace(text):
|
||||
return re.sub(_whitespace_re, " ", text)
|
||||
|
||||
|
||||
def convert_to_ascii(text):
|
||||
return unidecode(text)
|
||||
|
||||
|
||||
def basic_cleaners(text):
|
||||
"""Basic pipeline that lowercases and collapses whitespace without transliteration."""
|
||||
text = lowercase(text)
|
||||
text = collapse_whitespace(text)
|
||||
return text
|
||||
|
||||
|
||||
def transliteration_cleaners(text):
|
||||
"""Pipeline for non-English text that transliterates to ASCII."""
|
||||
text = convert_to_ascii(text)
|
||||
text = lowercase(text)
|
||||
text = collapse_whitespace(text)
|
||||
return text
|
||||
|
||||
|
||||
def english_cleaners2(text):
|
||||
"""Pipeline for English text, including abbreviation expansion. + punctuation + stress"""
|
||||
text = convert_to_ascii(text)
|
||||
text = lowercase(text)
|
||||
text = expand_abbreviations(text)
|
||||
phonemes = global_phonemizer.phonemize([text], strip=True, njobs=1)[0]
|
||||
phonemes = collapse_whitespace(phonemes)
|
||||
return phonemes
|
||||
|
||||
|
||||
def english_cleaners_piper(text):
|
||||
"""Pipeline for English text, including abbreviation expansion. + punctuation + stress"""
|
||||
text = convert_to_ascii(text)
|
||||
text = lowercase(text)
|
||||
text = expand_abbreviations(text)
|
||||
phonemes = "".join(piper_phonemize.phonemize_espeak(text=text, voice="en-US")[0])
|
||||
phonemes = collapse_whitespace(phonemes)
|
||||
return phonemes
|
||||
71
vendor/CosyVoice/third_party/Matcha-TTS/matcha/text/numbers.py
vendored
Normal file
71
vendor/CosyVoice/third_party/Matcha-TTS/matcha/text/numbers.py
vendored
Normal file
@@ -0,0 +1,71 @@
|
||||
""" from https://github.com/keithito/tacotron """
|
||||
|
||||
import re
|
||||
|
||||
import inflect
|
||||
|
||||
_inflect = inflect.engine()
|
||||
_comma_number_re = re.compile(r"([0-9][0-9\,]+[0-9])")
|
||||
_decimal_number_re = re.compile(r"([0-9]+\.[0-9]+)")
|
||||
_pounds_re = re.compile(r"£([0-9\,]*[0-9]+)")
|
||||
_dollars_re = re.compile(r"\$([0-9\.\,]*[0-9]+)")
|
||||
_ordinal_re = re.compile(r"[0-9]+(st|nd|rd|th)")
|
||||
_number_re = re.compile(r"[0-9]+")
|
||||
|
||||
|
||||
def _remove_commas(m):
|
||||
return m.group(1).replace(",", "")
|
||||
|
||||
|
||||
def _expand_decimal_point(m):
|
||||
return m.group(1).replace(".", " point ")
|
||||
|
||||
|
||||
def _expand_dollars(m):
|
||||
match = m.group(1)
|
||||
parts = match.split(".")
|
||||
if len(parts) > 2:
|
||||
return match + " dollars"
|
||||
dollars = int(parts[0]) if parts[0] else 0
|
||||
cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0
|
||||
if dollars and cents:
|
||||
dollar_unit = "dollar" if dollars == 1 else "dollars"
|
||||
cent_unit = "cent" if cents == 1 else "cents"
|
||||
return f"{dollars} {dollar_unit}, {cents} {cent_unit}"
|
||||
elif dollars:
|
||||
dollar_unit = "dollar" if dollars == 1 else "dollars"
|
||||
return f"{dollars} {dollar_unit}"
|
||||
elif cents:
|
||||
cent_unit = "cent" if cents == 1 else "cents"
|
||||
return f"{cents} {cent_unit}"
|
||||
else:
|
||||
return "zero dollars"
|
||||
|
||||
|
||||
def _expand_ordinal(m):
|
||||
return _inflect.number_to_words(m.group(0))
|
||||
|
||||
|
||||
def _expand_number(m):
|
||||
num = int(m.group(0))
|
||||
if num > 1000 and num < 3000:
|
||||
if num == 2000:
|
||||
return "two thousand"
|
||||
elif num > 2000 and num < 2010:
|
||||
return "two thousand " + _inflect.number_to_words(num % 100)
|
||||
elif num % 100 == 0:
|
||||
return _inflect.number_to_words(num // 100) + " hundred"
|
||||
else:
|
||||
return _inflect.number_to_words(num, andword="", zero="oh", group=2).replace(", ", " ")
|
||||
else:
|
||||
return _inflect.number_to_words(num, andword="")
|
||||
|
||||
|
||||
def normalize_numbers(text):
|
||||
text = re.sub(_comma_number_re, _remove_commas, text)
|
||||
text = re.sub(_pounds_re, r"\1 pounds", text)
|
||||
text = re.sub(_dollars_re, _expand_dollars, text)
|
||||
text = re.sub(_decimal_number_re, _expand_decimal_point, text)
|
||||
text = re.sub(_ordinal_re, _expand_ordinal, text)
|
||||
text = re.sub(_number_re, _expand_number, text)
|
||||
return text
|
||||
17
vendor/CosyVoice/third_party/Matcha-TTS/matcha/text/symbols.py
vendored
Normal file
17
vendor/CosyVoice/third_party/Matcha-TTS/matcha/text/symbols.py
vendored
Normal file
@@ -0,0 +1,17 @@
|
||||
""" from https://github.com/keithito/tacotron
|
||||
|
||||
Defines the set of symbols used in text input to the model.
|
||||
"""
|
||||
_pad = "_"
|
||||
_punctuation = ';:,.!?¡¿—…"«»“” '
|
||||
_letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
|
||||
_letters_ipa = (
|
||||
"ɑɐɒæɓʙβɔɕçɗɖðʤəɘɚɛɜɝɞɟʄɡɠɢʛɦɧħɥʜɨɪʝɭɬɫɮʟɱɯɰŋɳɲɴøɵɸθœɶʘɹɺɾɻʀʁɽʂʃʈʧʉʊʋⱱʌɣɤʍχʎʏʑʐʒʔʡʕʢǀǁǂǃˈˌːˑʼʴʰʱʲʷˠˤ˞↓↑→↗↘'̩'ᵻ"
|
||||
)
|
||||
|
||||
|
||||
# Export all symbols:
|
||||
symbols = [_pad] + list(_punctuation) + list(_letters) + list(_letters_ipa)
|
||||
|
||||
# Special symbol ids
|
||||
SPACE_ID = symbols.index(" ")
|
||||
122
vendor/CosyVoice/third_party/Matcha-TTS/matcha/train.py
vendored
Normal file
122
vendor/CosyVoice/third_party/Matcha-TTS/matcha/train.py
vendored
Normal file
@@ -0,0 +1,122 @@
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import hydra
|
||||
import lightning as L
|
||||
import rootutils
|
||||
from lightning import Callback, LightningDataModule, LightningModule, Trainer
|
||||
from lightning.pytorch.loggers import Logger
|
||||
from omegaconf import DictConfig
|
||||
|
||||
from matcha import utils
|
||||
|
||||
rootutils.setup_root(__file__, indicator=".project-root", pythonpath=True)
|
||||
# ------------------------------------------------------------------------------------ #
|
||||
# the setup_root above is equivalent to:
|
||||
# - adding project root dir to PYTHONPATH
|
||||
# (so you don't need to force user to install project as a package)
|
||||
# (necessary before importing any local modules e.g. `from src import utils`)
|
||||
# - setting up PROJECT_ROOT environment variable
|
||||
# (which is used as a base for paths in "configs/paths/default.yaml")
|
||||
# (this way all filepaths are the same no matter where you run the code)
|
||||
# - loading environment variables from ".env" in root dir
|
||||
#
|
||||
# you can remove it if you:
|
||||
# 1. either install project as a package or move entry files to project root dir
|
||||
# 2. set `root_dir` to "." in "configs/paths/default.yaml"
|
||||
#
|
||||
# more info: https://github.com/ashleve/rootutils
|
||||
# ------------------------------------------------------------------------------------ #
|
||||
|
||||
|
||||
log = utils.get_pylogger(__name__)
|
||||
|
||||
|
||||
@utils.task_wrapper
|
||||
def train(cfg: DictConfig) -> Tuple[Dict[str, Any], Dict[str, Any]]:
|
||||
"""Trains the model. Can additionally evaluate on a testset, using best weights obtained during
|
||||
training.
|
||||
|
||||
This method is wrapped in optional @task_wrapper decorator, that controls the behavior during
|
||||
failure. Useful for multiruns, saving info about the crash, etc.
|
||||
|
||||
:param cfg: A DictConfig configuration composed by Hydra.
|
||||
:return: A tuple with metrics and dict with all instantiated objects.
|
||||
"""
|
||||
# set seed for random number generators in pytorch, numpy and python.random
|
||||
if cfg.get("seed"):
|
||||
L.seed_everything(cfg.seed, workers=True)
|
||||
|
||||
log.info(f"Instantiating datamodule <{cfg.data._target_}>") # pylint: disable=protected-access
|
||||
datamodule: LightningDataModule = hydra.utils.instantiate(cfg.data)
|
||||
|
||||
log.info(f"Instantiating model <{cfg.model._target_}>") # pylint: disable=protected-access
|
||||
model: LightningModule = hydra.utils.instantiate(cfg.model)
|
||||
|
||||
log.info("Instantiating callbacks...")
|
||||
callbacks: List[Callback] = utils.instantiate_callbacks(cfg.get("callbacks"))
|
||||
|
||||
log.info("Instantiating loggers...")
|
||||
logger: List[Logger] = utils.instantiate_loggers(cfg.get("logger"))
|
||||
|
||||
log.info(f"Instantiating trainer <{cfg.trainer._target_}>") # pylint: disable=protected-access
|
||||
trainer: Trainer = hydra.utils.instantiate(cfg.trainer, callbacks=callbacks, logger=logger)
|
||||
|
||||
object_dict = {
|
||||
"cfg": cfg,
|
||||
"datamodule": datamodule,
|
||||
"model": model,
|
||||
"callbacks": callbacks,
|
||||
"logger": logger,
|
||||
"trainer": trainer,
|
||||
}
|
||||
|
||||
if logger:
|
||||
log.info("Logging hyperparameters!")
|
||||
utils.log_hyperparameters(object_dict)
|
||||
|
||||
if cfg.get("train"):
|
||||
log.info("Starting training!")
|
||||
trainer.fit(model=model, datamodule=datamodule, ckpt_path=cfg.get("ckpt_path"))
|
||||
|
||||
train_metrics = trainer.callback_metrics
|
||||
|
||||
if cfg.get("test"):
|
||||
log.info("Starting testing!")
|
||||
ckpt_path = trainer.checkpoint_callback.best_model_path
|
||||
if ckpt_path == "":
|
||||
log.warning("Best ckpt not found! Using current weights for testing...")
|
||||
ckpt_path = None
|
||||
trainer.test(model=model, datamodule=datamodule, ckpt_path=ckpt_path)
|
||||
log.info(f"Best ckpt path: {ckpt_path}")
|
||||
|
||||
test_metrics = trainer.callback_metrics
|
||||
|
||||
# merge train and test metrics
|
||||
metric_dict = {**train_metrics, **test_metrics}
|
||||
|
||||
return metric_dict, object_dict
|
||||
|
||||
|
||||
@hydra.main(version_base="1.3", config_path="../configs", config_name="train.yaml")
|
||||
def main(cfg: DictConfig) -> Optional[float]:
|
||||
"""Main entry point for training.
|
||||
|
||||
:param cfg: DictConfig configuration composed by Hydra.
|
||||
:return: Optional[float] with optimized metric value.
|
||||
"""
|
||||
# apply extra utilities
|
||||
# (e.g. ask for tags if none are provided in cfg, print cfg tree, etc.)
|
||||
utils.extras(cfg)
|
||||
|
||||
# train the model
|
||||
metric_dict, _ = train(cfg)
|
||||
|
||||
# safely retrieve metric value for hydra-based hyperparameter optimization
|
||||
metric_value = utils.get_metric_value(metric_dict=metric_dict, metric_name=cfg.get("optimized_metric"))
|
||||
|
||||
# return optimized metric
|
||||
return metric_value
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main() # pylint: disable=no-value-for-parameter
|
||||
5
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/__init__.py
vendored
Normal file
5
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/__init__.py
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
from matcha.utils.instantiators import instantiate_callbacks, instantiate_loggers
|
||||
from matcha.utils.logging_utils import log_hyperparameters
|
||||
from matcha.utils.pylogger import get_pylogger
|
||||
from matcha.utils.rich_utils import enforce_tags, print_config_tree
|
||||
from matcha.utils.utils import extras, get_metric_value, task_wrapper
|
||||
82
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/audio.py
vendored
Normal file
82
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/audio.py
vendored
Normal file
@@ -0,0 +1,82 @@
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.utils.data
|
||||
from librosa.filters import mel as librosa_mel_fn
|
||||
from scipy.io.wavfile import read
|
||||
|
||||
MAX_WAV_VALUE = 32768.0
|
||||
|
||||
|
||||
def load_wav(full_path):
|
||||
sampling_rate, data = read(full_path)
|
||||
return data, sampling_rate
|
||||
|
||||
|
||||
def dynamic_range_compression(x, C=1, clip_val=1e-5):
|
||||
return np.log(np.clip(x, a_min=clip_val, a_max=None) * C)
|
||||
|
||||
|
||||
def dynamic_range_decompression(x, C=1):
|
||||
return np.exp(x) / C
|
||||
|
||||
|
||||
def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
|
||||
return torch.log(torch.clamp(x, min=clip_val) * C)
|
||||
|
||||
|
||||
def dynamic_range_decompression_torch(x, C=1):
|
||||
return torch.exp(x) / C
|
||||
|
||||
|
||||
def spectral_normalize_torch(magnitudes):
|
||||
output = dynamic_range_compression_torch(magnitudes)
|
||||
return output
|
||||
|
||||
|
||||
def spectral_de_normalize_torch(magnitudes):
|
||||
output = dynamic_range_decompression_torch(magnitudes)
|
||||
return output
|
||||
|
||||
|
||||
mel_basis = {}
|
||||
hann_window = {}
|
||||
|
||||
|
||||
def mel_spectrogram(y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False):
|
||||
if torch.min(y) < -1.0:
|
||||
print("min value is ", torch.min(y))
|
||||
if torch.max(y) > 1.0:
|
||||
print("max value is ", torch.max(y))
|
||||
|
||||
global mel_basis, hann_window # pylint: disable=global-statement
|
||||
if f"{str(fmax)}_{str(y.device)}" not in mel_basis:
|
||||
mel = librosa_mel_fn(sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax)
|
||||
mel_basis[str(fmax) + "_" + str(y.device)] = torch.from_numpy(mel).float().to(y.device)
|
||||
hann_window[str(y.device)] = torch.hann_window(win_size).to(y.device)
|
||||
|
||||
y = torch.nn.functional.pad(
|
||||
y.unsqueeze(1), (int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)), mode="reflect"
|
||||
)
|
||||
y = y.squeeze(1)
|
||||
|
||||
spec = torch.view_as_real(
|
||||
torch.stft(
|
||||
y,
|
||||
n_fft,
|
||||
hop_length=hop_size,
|
||||
win_length=win_size,
|
||||
window=hann_window[str(y.device)],
|
||||
center=center,
|
||||
pad_mode="reflect",
|
||||
normalized=False,
|
||||
onesided=True,
|
||||
return_complex=True,
|
||||
)
|
||||
)
|
||||
|
||||
spec = torch.sqrt(spec.pow(2).sum(-1) + (1e-9))
|
||||
|
||||
spec = torch.matmul(mel_basis[str(fmax) + "_" + str(y.device)], spec)
|
||||
spec = spectral_normalize_torch(spec)
|
||||
|
||||
return spec
|
||||
111
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/generate_data_statistics.py
vendored
Normal file
111
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/generate_data_statistics.py
vendored
Normal file
@@ -0,0 +1,111 @@
|
||||
r"""
|
||||
The file creates a pickle file where the values needed for loading of dataset is stored and the model can load it
|
||||
when needed.
|
||||
|
||||
Parameters from hparam.py will be used
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import rootutils
|
||||
import torch
|
||||
from hydra import compose, initialize
|
||||
from omegaconf import open_dict
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
from matcha.data.text_mel_datamodule import TextMelDataModule
|
||||
from matcha.utils.logging_utils import pylogger
|
||||
|
||||
log = pylogger.get_pylogger(__name__)
|
||||
|
||||
|
||||
def compute_data_statistics(data_loader: torch.utils.data.DataLoader, out_channels: int):
|
||||
"""Generate data mean and standard deviation helpful in data normalisation
|
||||
|
||||
Args:
|
||||
data_loader (torch.utils.data.Dataloader): _description_
|
||||
out_channels (int): mel spectrogram channels
|
||||
"""
|
||||
total_mel_sum = 0
|
||||
total_mel_sq_sum = 0
|
||||
total_mel_len = 0
|
||||
|
||||
for batch in tqdm(data_loader, leave=False):
|
||||
mels = batch["y"]
|
||||
mel_lengths = batch["y_lengths"]
|
||||
|
||||
total_mel_len += torch.sum(mel_lengths)
|
||||
total_mel_sum += torch.sum(mels)
|
||||
total_mel_sq_sum += torch.sum(torch.pow(mels, 2))
|
||||
|
||||
data_mean = total_mel_sum / (total_mel_len * out_channels)
|
||||
data_std = torch.sqrt((total_mel_sq_sum / (total_mel_len * out_channels)) - torch.pow(data_mean, 2))
|
||||
|
||||
return {"mel_mean": data_mean.item(), "mel_std": data_std.item()}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument(
|
||||
"-i",
|
||||
"--input-config",
|
||||
type=str,
|
||||
default="vctk.yaml",
|
||||
help="The name of the yaml config file under configs/data",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-b",
|
||||
"--batch-size",
|
||||
type=int,
|
||||
default="256",
|
||||
help="Can have increased batch size for faster computation",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-f",
|
||||
"--force",
|
||||
action="store_true",
|
||||
default=False,
|
||||
required=False,
|
||||
help="force overwrite the file",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
output_file = Path(args.input_config).with_suffix(".json")
|
||||
|
||||
if os.path.exists(output_file) and not args.force:
|
||||
print("File already exists. Use -f to force overwrite")
|
||||
sys.exit(1)
|
||||
|
||||
with initialize(version_base="1.3", config_path="../../configs/data"):
|
||||
cfg = compose(config_name=args.input_config, return_hydra_config=True, overrides=[])
|
||||
|
||||
root_path = rootutils.find_root(search_from=__file__, indicator=".project-root")
|
||||
|
||||
with open_dict(cfg):
|
||||
del cfg["hydra"]
|
||||
del cfg["_target_"]
|
||||
cfg["data_statistics"] = None
|
||||
cfg["seed"] = 1234
|
||||
cfg["batch_size"] = args.batch_size
|
||||
cfg["train_filelist_path"] = str(os.path.join(root_path, cfg["train_filelist_path"]))
|
||||
cfg["valid_filelist_path"] = str(os.path.join(root_path, cfg["valid_filelist_path"]))
|
||||
|
||||
text_mel_datamodule = TextMelDataModule(**cfg)
|
||||
text_mel_datamodule.setup()
|
||||
data_loader = text_mel_datamodule.train_dataloader()
|
||||
log.info("Dataloader loaded! Now computing stats...")
|
||||
params = compute_data_statistics(data_loader, cfg["n_feats"])
|
||||
print(params)
|
||||
json.dump(
|
||||
params,
|
||||
open(output_file, "w"),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
56
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/instantiators.py
vendored
Normal file
56
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/instantiators.py
vendored
Normal file
@@ -0,0 +1,56 @@
|
||||
from typing import List
|
||||
|
||||
import hydra
|
||||
from lightning import Callback
|
||||
from lightning.pytorch.loggers import Logger
|
||||
from omegaconf import DictConfig
|
||||
|
||||
from matcha.utils import pylogger
|
||||
|
||||
log = pylogger.get_pylogger(__name__)
|
||||
|
||||
|
||||
def instantiate_callbacks(callbacks_cfg: DictConfig) -> List[Callback]:
|
||||
"""Instantiates callbacks from config.
|
||||
|
||||
:param callbacks_cfg: A DictConfig object containing callback configurations.
|
||||
:return: A list of instantiated callbacks.
|
||||
"""
|
||||
callbacks: List[Callback] = []
|
||||
|
||||
if not callbacks_cfg:
|
||||
log.warning("No callback configs found! Skipping..")
|
||||
return callbacks
|
||||
|
||||
if not isinstance(callbacks_cfg, DictConfig):
|
||||
raise TypeError("Callbacks config must be a DictConfig!")
|
||||
|
||||
for _, cb_conf in callbacks_cfg.items():
|
||||
if isinstance(cb_conf, DictConfig) and "_target_" in cb_conf:
|
||||
log.info(f"Instantiating callback <{cb_conf._target_}>") # pylint: disable=protected-access
|
||||
callbacks.append(hydra.utils.instantiate(cb_conf))
|
||||
|
||||
return callbacks
|
||||
|
||||
|
||||
def instantiate_loggers(logger_cfg: DictConfig) -> List[Logger]:
|
||||
"""Instantiates loggers from config.
|
||||
|
||||
:param logger_cfg: A DictConfig object containing logger configurations.
|
||||
:return: A list of instantiated loggers.
|
||||
"""
|
||||
logger: List[Logger] = []
|
||||
|
||||
if not logger_cfg:
|
||||
log.warning("No logger configs found! Skipping...")
|
||||
return logger
|
||||
|
||||
if not isinstance(logger_cfg, DictConfig):
|
||||
raise TypeError("Logger config must be a DictConfig!")
|
||||
|
||||
for _, lg_conf in logger_cfg.items():
|
||||
if isinstance(lg_conf, DictConfig) and "_target_" in lg_conf:
|
||||
log.info(f"Instantiating logger <{lg_conf._target_}>") # pylint: disable=protected-access
|
||||
logger.append(hydra.utils.instantiate(lg_conf))
|
||||
|
||||
return logger
|
||||
53
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/logging_utils.py
vendored
Normal file
53
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/logging_utils.py
vendored
Normal file
@@ -0,0 +1,53 @@
|
||||
from typing import Any, Dict
|
||||
|
||||
from lightning.pytorch.utilities import rank_zero_only
|
||||
from omegaconf import OmegaConf
|
||||
|
||||
from matcha.utils import pylogger
|
||||
|
||||
log = pylogger.get_pylogger(__name__)
|
||||
|
||||
|
||||
@rank_zero_only
|
||||
def log_hyperparameters(object_dict: Dict[str, Any]) -> None:
|
||||
"""Controls which config parts are saved by Lightning loggers.
|
||||
|
||||
Additionally saves:
|
||||
- Number of model parameters
|
||||
|
||||
:param object_dict: A dictionary containing the following objects:
|
||||
- `"cfg"`: A DictConfig object containing the main config.
|
||||
- `"model"`: The Lightning model.
|
||||
- `"trainer"`: The Lightning trainer.
|
||||
"""
|
||||
hparams = {}
|
||||
|
||||
cfg = OmegaConf.to_container(object_dict["cfg"])
|
||||
model = object_dict["model"]
|
||||
trainer = object_dict["trainer"]
|
||||
|
||||
if not trainer.logger:
|
||||
log.warning("Logger not found! Skipping hyperparameter logging...")
|
||||
return
|
||||
|
||||
hparams["model"] = cfg["model"]
|
||||
|
||||
# save number of model parameters
|
||||
hparams["model/params/total"] = sum(p.numel() for p in model.parameters())
|
||||
hparams["model/params/trainable"] = sum(p.numel() for p in model.parameters() if p.requires_grad)
|
||||
hparams["model/params/non_trainable"] = sum(p.numel() for p in model.parameters() if not p.requires_grad)
|
||||
|
||||
hparams["data"] = cfg["data"]
|
||||
hparams["trainer"] = cfg["trainer"]
|
||||
|
||||
hparams["callbacks"] = cfg.get("callbacks")
|
||||
hparams["extras"] = cfg.get("extras")
|
||||
|
||||
hparams["task_name"] = cfg.get("task_name")
|
||||
hparams["tags"] = cfg.get("tags")
|
||||
hparams["ckpt_path"] = cfg.get("ckpt_path")
|
||||
hparams["seed"] = cfg.get("seed")
|
||||
|
||||
# send hparams to all loggers
|
||||
for logger in trainer.loggers:
|
||||
logger.log_hyperparams(hparams)
|
||||
90
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/model.py
vendored
Normal file
90
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/model.py
vendored
Normal file
@@ -0,0 +1,90 @@
|
||||
""" from https://github.com/jaywalnut310/glow-tts """
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
|
||||
def sequence_mask(length, max_length=None):
|
||||
if max_length is None:
|
||||
max_length = length.max()
|
||||
x = torch.arange(max_length, dtype=length.dtype, device=length.device)
|
||||
return x.unsqueeze(0) < length.unsqueeze(1)
|
||||
|
||||
|
||||
def fix_len_compatibility(length, num_downsamplings_in_unet=2):
|
||||
factor = torch.scalar_tensor(2).pow(num_downsamplings_in_unet)
|
||||
length = (length / factor).ceil() * factor
|
||||
if not torch.onnx.is_in_onnx_export():
|
||||
return length.int().item()
|
||||
else:
|
||||
return length
|
||||
|
||||
|
||||
def convert_pad_shape(pad_shape):
|
||||
inverted_shape = pad_shape[::-1]
|
||||
pad_shape = [item for sublist in inverted_shape for item in sublist]
|
||||
return pad_shape
|
||||
|
||||
|
||||
def generate_path(duration, mask):
|
||||
device = duration.device
|
||||
|
||||
b, t_x, t_y = mask.shape
|
||||
cum_duration = torch.cumsum(duration, 1)
|
||||
path = torch.zeros(b, t_x, t_y, dtype=mask.dtype).to(device=device)
|
||||
|
||||
cum_duration_flat = cum_duration.view(b * t_x)
|
||||
path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
|
||||
path = path.view(b, t_x, t_y)
|
||||
path = path - torch.nn.functional.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1]
|
||||
path = path * mask
|
||||
return path
|
||||
|
||||
|
||||
def duration_loss(logw, logw_, lengths):
|
||||
loss = torch.sum((logw - logw_) ** 2) / torch.sum(lengths)
|
||||
return loss
|
||||
|
||||
|
||||
def normalize(data, mu, std):
|
||||
if not isinstance(mu, (float, int)):
|
||||
if isinstance(mu, list):
|
||||
mu = torch.tensor(mu, dtype=data.dtype, device=data.device)
|
||||
elif isinstance(mu, torch.Tensor):
|
||||
mu = mu.to(data.device)
|
||||
elif isinstance(mu, np.ndarray):
|
||||
mu = torch.from_numpy(mu).to(data.device)
|
||||
mu = mu.unsqueeze(-1)
|
||||
|
||||
if not isinstance(std, (float, int)):
|
||||
if isinstance(std, list):
|
||||
std = torch.tensor(std, dtype=data.dtype, device=data.device)
|
||||
elif isinstance(std, torch.Tensor):
|
||||
std = std.to(data.device)
|
||||
elif isinstance(std, np.ndarray):
|
||||
std = torch.from_numpy(std).to(data.device)
|
||||
std = std.unsqueeze(-1)
|
||||
|
||||
return (data - mu) / std
|
||||
|
||||
|
||||
def denormalize(data, mu, std):
|
||||
if not isinstance(mu, float):
|
||||
if isinstance(mu, list):
|
||||
mu = torch.tensor(mu, dtype=data.dtype, device=data.device)
|
||||
elif isinstance(mu, torch.Tensor):
|
||||
mu = mu.to(data.device)
|
||||
elif isinstance(mu, np.ndarray):
|
||||
mu = torch.from_numpy(mu).to(data.device)
|
||||
mu = mu.unsqueeze(-1)
|
||||
|
||||
if not isinstance(std, float):
|
||||
if isinstance(std, list):
|
||||
std = torch.tensor(std, dtype=data.dtype, device=data.device)
|
||||
elif isinstance(std, torch.Tensor):
|
||||
std = std.to(data.device)
|
||||
elif isinstance(std, np.ndarray):
|
||||
std = torch.from_numpy(std).to(data.device)
|
||||
std = std.unsqueeze(-1)
|
||||
|
||||
return data * std + mu
|
||||
22
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/monotonic_align/__init__.py
vendored
Normal file
22
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/monotonic_align/__init__.py
vendored
Normal file
@@ -0,0 +1,22 @@
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from matcha.utils.monotonic_align.core import maximum_path_c
|
||||
|
||||
|
||||
def maximum_path(value, mask):
|
||||
"""Cython optimised version.
|
||||
value: [b, t_x, t_y]
|
||||
mask: [b, t_x, t_y]
|
||||
"""
|
||||
value = value * mask
|
||||
device = value.device
|
||||
dtype = value.dtype
|
||||
value = value.data.cpu().numpy().astype(np.float32)
|
||||
path = np.zeros_like(value).astype(np.int32)
|
||||
mask = mask.data.cpu().numpy()
|
||||
|
||||
t_x_max = mask.sum(1)[:, 0].astype(np.int32)
|
||||
t_y_max = mask.sum(2)[:, 0].astype(np.int32)
|
||||
maximum_path_c(path, value, t_x_max, t_y_max)
|
||||
return torch.from_numpy(path).to(device=device, dtype=dtype)
|
||||
47
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/monotonic_align/core.pyx
vendored
Normal file
47
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/monotonic_align/core.pyx
vendored
Normal file
@@ -0,0 +1,47 @@
|
||||
import numpy as np
|
||||
|
||||
cimport cython
|
||||
cimport numpy as np
|
||||
|
||||
from cython.parallel import prange
|
||||
|
||||
|
||||
@cython.boundscheck(False)
|
||||
@cython.wraparound(False)
|
||||
cdef void maximum_path_each(int[:,::1] path, float[:,::1] value, int t_x, int t_y, float max_neg_val) nogil:
|
||||
cdef int x
|
||||
cdef int y
|
||||
cdef float v_prev
|
||||
cdef float v_cur
|
||||
cdef float tmp
|
||||
cdef int index = t_x - 1
|
||||
|
||||
for y in range(t_y):
|
||||
for x in range(max(0, t_x + y - t_y), min(t_x, y + 1)):
|
||||
if x == y:
|
||||
v_cur = max_neg_val
|
||||
else:
|
||||
v_cur = value[x, y-1]
|
||||
if x == 0:
|
||||
if y == 0:
|
||||
v_prev = 0.
|
||||
else:
|
||||
v_prev = max_neg_val
|
||||
else:
|
||||
v_prev = value[x-1, y-1]
|
||||
value[x, y] = max(v_cur, v_prev) + value[x, y]
|
||||
|
||||
for y in range(t_y - 1, -1, -1):
|
||||
path[index, y] = 1
|
||||
if index != 0 and (index == y or value[index, y-1] < value[index-1, y-1]):
|
||||
index = index - 1
|
||||
|
||||
|
||||
@cython.boundscheck(False)
|
||||
@cython.wraparound(False)
|
||||
cpdef void maximum_path_c(int[:,:,::1] paths, float[:,:,::1] values, int[::1] t_xs, int[::1] t_ys, float max_neg_val=-1e9) nogil:
|
||||
cdef int b = values.shape[0]
|
||||
|
||||
cdef int i
|
||||
for i in prange(b, nogil=True):
|
||||
maximum_path_each(paths[i], values[i], t_xs[i], t_ys[i], max_neg_val)
|
||||
7
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/monotonic_align/setup.py
vendored
Normal file
7
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/monotonic_align/setup.py
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
# from distutils.core import setup
|
||||
# from Cython.Build import cythonize
|
||||
# import numpy
|
||||
|
||||
# setup(name='monotonic_align',
|
||||
# ext_modules=cythonize("core.pyx"),
|
||||
# include_dirs=[numpy.get_include()])
|
||||
21
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/pylogger.py
vendored
Normal file
21
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/pylogger.py
vendored
Normal file
@@ -0,0 +1,21 @@
|
||||
import logging
|
||||
|
||||
from lightning.pytorch.utilities import rank_zero_only
|
||||
|
||||
|
||||
def get_pylogger(name: str = __name__) -> logging.Logger:
|
||||
"""Initializes a multi-GPU-friendly python command line logger.
|
||||
|
||||
:param name: The name of the logger, defaults to ``__name__``.
|
||||
|
||||
:return: A logger object.
|
||||
"""
|
||||
logger = logging.getLogger(name)
|
||||
|
||||
# this ensures all logging levels get marked with the rank zero decorator
|
||||
# otherwise logs would get multiplied for each GPU process in multi-GPU setup
|
||||
logging_levels = ("debug", "info", "warning", "error", "exception", "fatal", "critical")
|
||||
for level in logging_levels:
|
||||
setattr(logger, level, rank_zero_only(getattr(logger, level)))
|
||||
|
||||
return logger
|
||||
101
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/rich_utils.py
vendored
Normal file
101
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/rich_utils.py
vendored
Normal file
@@ -0,0 +1,101 @@
|
||||
from pathlib import Path
|
||||
from typing import Sequence
|
||||
|
||||
import rich
|
||||
import rich.syntax
|
||||
import rich.tree
|
||||
from hydra.core.hydra_config import HydraConfig
|
||||
from lightning.pytorch.utilities import rank_zero_only
|
||||
from omegaconf import DictConfig, OmegaConf, open_dict
|
||||
from rich.prompt import Prompt
|
||||
|
||||
from matcha.utils import pylogger
|
||||
|
||||
log = pylogger.get_pylogger(__name__)
|
||||
|
||||
|
||||
@rank_zero_only
|
||||
def print_config_tree(
|
||||
cfg: DictConfig,
|
||||
print_order: Sequence[str] = (
|
||||
"data",
|
||||
"model",
|
||||
"callbacks",
|
||||
"logger",
|
||||
"trainer",
|
||||
"paths",
|
||||
"extras",
|
||||
),
|
||||
resolve: bool = False,
|
||||
save_to_file: bool = False,
|
||||
) -> None:
|
||||
"""Prints the contents of a DictConfig as a tree structure using the Rich library.
|
||||
|
||||
:param cfg: A DictConfig composed by Hydra.
|
||||
:param print_order: Determines in what order config components are printed. Default is ``("data", "model",
|
||||
"callbacks", "logger", "trainer", "paths", "extras")``.
|
||||
:param resolve: Whether to resolve reference fields of DictConfig. Default is ``False``.
|
||||
:param save_to_file: Whether to export config to the hydra output folder. Default is ``False``.
|
||||
"""
|
||||
style = "dim"
|
||||
tree = rich.tree.Tree("CONFIG", style=style, guide_style=style)
|
||||
|
||||
queue = []
|
||||
|
||||
# add fields from `print_order` to queue
|
||||
for field in print_order:
|
||||
_ = (
|
||||
queue.append(field)
|
||||
if field in cfg
|
||||
else log.warning(f"Field '{field}' not found in config. Skipping '{field}' config printing...")
|
||||
)
|
||||
|
||||
# add all the other fields to queue (not specified in `print_order`)
|
||||
for field in cfg:
|
||||
if field not in queue:
|
||||
queue.append(field)
|
||||
|
||||
# generate config tree from queue
|
||||
for field in queue:
|
||||
branch = tree.add(field, style=style, guide_style=style)
|
||||
|
||||
config_group = cfg[field]
|
||||
if isinstance(config_group, DictConfig):
|
||||
branch_content = OmegaConf.to_yaml(config_group, resolve=resolve)
|
||||
else:
|
||||
branch_content = str(config_group)
|
||||
|
||||
branch.add(rich.syntax.Syntax(branch_content, "yaml"))
|
||||
|
||||
# print config tree
|
||||
rich.print(tree)
|
||||
|
||||
# save config tree to file
|
||||
if save_to_file:
|
||||
with open(Path(cfg.paths.output_dir, "config_tree.log"), "w") as file:
|
||||
rich.print(tree, file=file)
|
||||
|
||||
|
||||
@rank_zero_only
|
||||
def enforce_tags(cfg: DictConfig, save_to_file: bool = False) -> None:
|
||||
"""Prompts user to input tags from command line if no tags are provided in config.
|
||||
|
||||
:param cfg: A DictConfig composed by Hydra.
|
||||
:param save_to_file: Whether to export tags to the hydra output folder. Default is ``False``.
|
||||
"""
|
||||
if not cfg.get("tags"):
|
||||
if "id" in HydraConfig().cfg.hydra.job:
|
||||
raise ValueError("Specify tags before launching a multirun!")
|
||||
|
||||
log.warning("No tags provided in config. Prompting user to input tags...")
|
||||
tags = Prompt.ask("Enter a list of comma separated tags", default="dev")
|
||||
tags = [t.strip() for t in tags.split(",") if t != ""]
|
||||
|
||||
with open_dict(cfg):
|
||||
cfg.tags = tags
|
||||
|
||||
log.info(f"Tags: {cfg.tags}")
|
||||
|
||||
if save_to_file:
|
||||
with open(Path(cfg.paths.output_dir, "tags.log"), "w") as file:
|
||||
rich.print(cfg.tags, file=file)
|
||||
219
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/utils.py
vendored
Normal file
219
vendor/CosyVoice/third_party/Matcha-TTS/matcha/utils/utils.py
vendored
Normal file
@@ -0,0 +1,219 @@
|
||||
import os
|
||||
import sys
|
||||
import warnings
|
||||
from importlib.util import find_spec
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, Tuple
|
||||
|
||||
import gdown
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import torch
|
||||
import wget
|
||||
from omegaconf import DictConfig
|
||||
|
||||
from matcha.utils import pylogger, rich_utils
|
||||
|
||||
log = pylogger.get_pylogger(__name__)
|
||||
|
||||
|
||||
def extras(cfg: DictConfig) -> None:
|
||||
"""Applies optional utilities before the task is started.
|
||||
|
||||
Utilities:
|
||||
- Ignoring python warnings
|
||||
- Setting tags from command line
|
||||
- Rich config printing
|
||||
|
||||
:param cfg: A DictConfig object containing the config tree.
|
||||
"""
|
||||
# return if no `extras` config
|
||||
if not cfg.get("extras"):
|
||||
log.warning("Extras config not found! <cfg.extras=null>")
|
||||
return
|
||||
|
||||
# disable python warnings
|
||||
if cfg.extras.get("ignore_warnings"):
|
||||
log.info("Disabling python warnings! <cfg.extras.ignore_warnings=True>")
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
# prompt user to input tags from command line if none are provided in the config
|
||||
if cfg.extras.get("enforce_tags"):
|
||||
log.info("Enforcing tags! <cfg.extras.enforce_tags=True>")
|
||||
rich_utils.enforce_tags(cfg, save_to_file=True)
|
||||
|
||||
# pretty print config tree using Rich library
|
||||
if cfg.extras.get("print_config"):
|
||||
log.info("Printing config tree with Rich! <cfg.extras.print_config=True>")
|
||||
rich_utils.print_config_tree(cfg, resolve=True, save_to_file=True)
|
||||
|
||||
|
||||
def task_wrapper(task_func: Callable) -> Callable:
|
||||
"""Optional decorator that controls the failure behavior when executing the task function.
|
||||
|
||||
This wrapper can be used to:
|
||||
- make sure loggers are closed even if the task function raises an exception (prevents multirun failure)
|
||||
- save the exception to a `.log` file
|
||||
- mark the run as failed with a dedicated file in the `logs/` folder (so we can find and rerun it later)
|
||||
- etc. (adjust depending on your needs)
|
||||
|
||||
Example:
|
||||
```
|
||||
@utils.task_wrapper
|
||||
def train(cfg: DictConfig) -> Tuple[Dict[str, Any], Dict[str, Any]]:
|
||||
...
|
||||
return metric_dict, object_dict
|
||||
```
|
||||
|
||||
:param task_func: The task function to be wrapped.
|
||||
|
||||
:return: The wrapped task function.
|
||||
"""
|
||||
|
||||
def wrap(cfg: DictConfig) -> Tuple[Dict[str, Any], Dict[str, Any]]:
|
||||
# execute the task
|
||||
try:
|
||||
metric_dict, object_dict = task_func(cfg=cfg)
|
||||
|
||||
# things to do if exception occurs
|
||||
except Exception as ex:
|
||||
# save exception to `.log` file
|
||||
log.exception("")
|
||||
|
||||
# some hyperparameter combinations might be invalid or cause out-of-memory errors
|
||||
# so when using hparam search plugins like Optuna, you might want to disable
|
||||
# raising the below exception to avoid multirun failure
|
||||
raise ex
|
||||
|
||||
# things to always do after either success or exception
|
||||
finally:
|
||||
# display output dir path in terminal
|
||||
log.info(f"Output dir: {cfg.paths.output_dir}")
|
||||
|
||||
# always close wandb run (even if exception occurs so multirun won't fail)
|
||||
if find_spec("wandb"): # check if wandb is installed
|
||||
import wandb
|
||||
|
||||
if wandb.run:
|
||||
log.info("Closing wandb!")
|
||||
wandb.finish()
|
||||
|
||||
return metric_dict, object_dict
|
||||
|
||||
return wrap
|
||||
|
||||
|
||||
def get_metric_value(metric_dict: Dict[str, Any], metric_name: str) -> float:
|
||||
"""Safely retrieves value of the metric logged in LightningModule.
|
||||
|
||||
:param metric_dict: A dict containing metric values.
|
||||
:param metric_name: The name of the metric to retrieve.
|
||||
:return: The value of the metric.
|
||||
"""
|
||||
if not metric_name:
|
||||
log.info("Metric name is None! Skipping metric value retrieval...")
|
||||
return None
|
||||
|
||||
if metric_name not in metric_dict:
|
||||
raise ValueError(
|
||||
f"Metric value not found! <metric_name={metric_name}>\n"
|
||||
"Make sure metric name logged in LightningModule is correct!\n"
|
||||
"Make sure `optimized_metric` name in `hparams_search` config is correct!"
|
||||
)
|
||||
|
||||
metric_value = metric_dict[metric_name].item()
|
||||
log.info(f"Retrieved metric value! <{metric_name}={metric_value}>")
|
||||
|
||||
return metric_value
|
||||
|
||||
|
||||
def intersperse(lst, item):
|
||||
# Adds blank symbol
|
||||
result = [item] * (len(lst) * 2 + 1)
|
||||
result[1::2] = lst
|
||||
return result
|
||||
|
||||
|
||||
def save_figure_to_numpy(fig):
|
||||
data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep="")
|
||||
data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
|
||||
return data
|
||||
|
||||
|
||||
def plot_tensor(tensor):
|
||||
plt.style.use("default")
|
||||
fig, ax = plt.subplots(figsize=(12, 3))
|
||||
im = ax.imshow(tensor, aspect="auto", origin="lower", interpolation="none")
|
||||
plt.colorbar(im, ax=ax)
|
||||
plt.tight_layout()
|
||||
fig.canvas.draw()
|
||||
data = save_figure_to_numpy(fig)
|
||||
plt.close()
|
||||
return data
|
||||
|
||||
|
||||
def save_plot(tensor, savepath):
|
||||
plt.style.use("default")
|
||||
fig, ax = plt.subplots(figsize=(12, 3))
|
||||
im = ax.imshow(tensor, aspect="auto", origin="lower", interpolation="none")
|
||||
plt.colorbar(im, ax=ax)
|
||||
plt.tight_layout()
|
||||
fig.canvas.draw()
|
||||
plt.savefig(savepath)
|
||||
plt.close()
|
||||
|
||||
|
||||
def to_numpy(tensor):
|
||||
if isinstance(tensor, np.ndarray):
|
||||
return tensor
|
||||
elif isinstance(tensor, torch.Tensor):
|
||||
return tensor.detach().cpu().numpy()
|
||||
elif isinstance(tensor, list):
|
||||
return np.array(tensor)
|
||||
else:
|
||||
raise TypeError("Unsupported type for conversion to numpy array")
|
||||
|
||||
|
||||
def get_user_data_dir(appname="matcha_tts"):
|
||||
"""
|
||||
Args:
|
||||
appname (str): Name of application
|
||||
|
||||
Returns:
|
||||
Path: path to user data directory
|
||||
"""
|
||||
|
||||
MATCHA_HOME = os.environ.get("MATCHA_HOME")
|
||||
if MATCHA_HOME is not None:
|
||||
ans = Path(MATCHA_HOME).expanduser().resolve(strict=False)
|
||||
elif sys.platform == "win32":
|
||||
import winreg # pylint: disable=import-outside-toplevel
|
||||
|
||||
key = winreg.OpenKey(
|
||||
winreg.HKEY_CURRENT_USER,
|
||||
r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders",
|
||||
)
|
||||
dir_, _ = winreg.QueryValueEx(key, "Local AppData")
|
||||
ans = Path(dir_).resolve(strict=False)
|
||||
elif sys.platform == "darwin":
|
||||
ans = Path("~/Library/Application Support/").expanduser()
|
||||
else:
|
||||
ans = Path.home().joinpath(".local/share")
|
||||
|
||||
final_path = ans.joinpath(appname)
|
||||
final_path.mkdir(parents=True, exist_ok=True)
|
||||
return final_path
|
||||
|
||||
|
||||
def assert_model_downloaded(checkpoint_path, url, use_wget=True):
|
||||
if Path(checkpoint_path).exists():
|
||||
log.debug(f"[+] Model already present at {checkpoint_path}!")
|
||||
print(f"[+] Model already present at {checkpoint_path}!")
|
||||
return
|
||||
log.info(f"[-] Model not found at {checkpoint_path}! Will download it")
|
||||
print(f"[-] Model not found at {checkpoint_path}! Will download it")
|
||||
checkpoint_path = str(checkpoint_path)
|
||||
if not use_wget:
|
||||
gdown.download(url=url, output=checkpoint_path, quiet=False, fuzzy=True)
|
||||
else:
|
||||
wget.download(url=url, out=checkpoint_path)
|
||||
Reference in New Issue
Block a user