"""Stage: CosyVoice2 (and Fun-CosyVoice3) Korean speed and samples. Runs in its own Python 3.10 venv. The CosyVoice source (pinned commit) and its Matcha-TTS submodule are unpacked by the orchestrator into /src. The voice prompt is one of the kit's synthetic Korean samples, so the output voice resembles MeloTTS; this measures speed and Korean pronunciation, not timbre. """ from __future__ import annotations import os import sys import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from m0lib.common import StageResult, gpu_memory_mib, say, stage_args, stats # noqa: E402 ASSETS = Path(__file__).resolve().parent.parent / "assets" / "ko_samples" MODELS = { "cosyvoice2": ("FunAudioLLM/CosyVoice2-0.5B", ""), "cosyvoice3": ("FunAudioLLM/Fun-CosyVoice3-0.5B-2512", "You are a helpful assistant.<|endofprompt|>"), } TEXTS = ["네, 알겠습니다. 지금 바로 틀어 드릴게요.", "오늘 서울은 맑고 낮 최고 기온은 십삼 도까지 오르겠습니다.", "회의는 오후 세 시에 시작해서 다섯 시쯤 끝날 예정이고, 끝난 뒤에는 저녁 메뉴를 같이 정하기로 하셨습니다."] def patch_load_wav() -> None: """torchaudio 2.9+ needs torchcodec for load(); read prompts with soundfile instead.""" import numpy as np import soundfile as sf import torch import torchaudio import cosyvoice.utils.file_utils as fu import cosyvoice.cli.frontend as fe def load_wav(wav, target_sr, min_sr=16000): data, sr = sf.read(wav, dtype="float32", always_2d=True) speech = torch.from_numpy(np.ascontiguousarray(data.mean(axis=1)))[None, :] if sr != target_sr: assert sr >= min_sr, f"wav sample rate {sr} must be at least {min_sr}" speech = torchaudio.transforms.Resample(orig_freq=sr, new_freq=target_sr)(speech) return speech fu.load_wav = load_wav fe.load_wav = load_wav def main() -> int: out, work, opts = stage_args() r = StageResult("tts_cosy", out) src = Path(opts.get("src", str(work / "src" / "CosyVoice"))) sys.path[:0] = [str(src), str(src / "third_party" / "Matcha-TTS")] os.environ.setdefault("HF_HOME", str(work / "models" / "hf")) os.environ.setdefault("MODELSCOPE_CACHE", str(work / "models" / "modelscope")) audio_dir = out.parent / "audio" audio_dir.mkdir(parents=True, exist_ok=True) try: import numpy as np import soundfile as sf import torch from huggingface_hub import snapshot_download from cosyvoice.cli.cosyvoice import AutoModel patch_load_wav() r["versions"] = {"torch": torch.__version__, "cuda_available": torch.cuda.is_available()} except Exception as e: r.error("import", e) r.save("failed") return 0 prompt_wav = str(ASSETS / "ko_08.wav") prompt_text = "오늘 회의는 오후 세 시에 시작해서 다섯 시쯤 끝날 것 같아요. 끝나고 나서 저녁 메뉴를 같이 정해 봐요." for key in opts.get("models", "cosyvoice2,cosyvoice3").split(","): repo, prefix = MODELS[key] res: dict = {"repo": repo} r[key] = res say(f" - {key}: 모델 받는 중") try: t0 = time.perf_counter() local = snapshot_download(repo, local_dir=str(work / "models" / "tts" / key)) res["download_s"] = round(time.perf_counter() - t0, 1) before = gpu_memory_mib() t0 = time.perf_counter() model = AutoModel(model_dir=local, fp16=torch.cuda.is_available()) res["load_s"] = round(time.perf_counter() - t0, 2) after = gpu_memory_mib() if before and after: res["vram_model_mib"] = after[0] - before[0] sr = model.sample_rate # Warm-up so CUDA start-up is not counted. for _ in model.inference_zero_shot("워밍업입니다.", prefix + prompt_text, prompt_wav, stream=False): pass say(f" 로딩 {res['load_s']}s, VRAM 약 {res.get('vram_model_mib')} MiB") except Exception as e: r.error(f"{key} load", e) r.save() continue rows = [] for ti, text in enumerate(TEXTS): for i in range(2): try: if torch.cuda.is_available(): torch.cuda.synchronize() t0 = time.perf_counter() first, chunks = None, [] for o in model.inference_zero_shot(text, prefix + prompt_text, prompt_wav, stream=True): if first is None: first = time.perf_counter() - t0 chunks.append(o["tts_speech"].squeeze(0).float().cpu().numpy()) total = time.perf_counter() - t0 wav = np.concatenate(chunks) if chunks else np.zeros(1, dtype=np.float32) dur = len(wav) / sr rows.append({"text": ti, "chars": len(text), "first_chunk_s": round(first or total, 3), "total_s": round(total, 3), "audio_s": round(dur, 2), "rtf": round(total / max(dur, 1e-3), 3), "try": i + 1}) if i == 0 and ti > 0: sf.write(str(audio_dir / f"{key}_{ti}.wav"), wav, sr) except Exception as e: r.error(f"{key} synth {ti}", e) res["rows"] = rows res["first_chunk"] = stats(x["first_chunk_s"] for x in rows) res["rtf"] = stats(x["rtf"] for x in rows) say(f" 첫 소리까지 중앙값 {res['first_chunk'].get('median')}s, RTF {res['rtf'].get('median')}") del model if torch.cuda.is_available(): torch.cuda.empty_cache() r.save() r["audio_files"] = sorted(p.name for p in audio_dir.glob("cosyvoice*.wav")) r.save("ok" if not r["errors"] else "partial") return 0 if __name__ == "__main__": sys.exit(main())