"""Per-stage Python environments, created with uv inside the work folder. Each heavy stage gets its own venv because their pins conflict (MeloTTS needs transformers 4.27 on Python 3.11, CosyVoice needs transformers 4.51, ...). """ from __future__ import annotations import io import os import shutil import sys import urllib.request import zipfile from pathlib import Path from m0lib.common import IS_WINDOWS, run_logged, say CU128_INDEX = "https://download.pytorch.org/whl/cu128" TORCH_CU128 = ["torch==2.11.0", "torchaudio==2.11.0", "--index-url", CU128_INDEX] # After the CUDA torch is in, later steps must not swap it for another build # (openai-whisper's Linux triton<3 pin silently downgraded torch to 2.3.1). TORCH_PIN = "torch==2.11.0+cu128\ntorchaudio==2.11.0+cu128\n" MELO_URL = "https://github.com/myshell-ai/MeloTTS/archive/209145371cff8fc3bd60d7be902ea69cbdb7965a.tar.gz" COSY_ZIP = "https://github.com/FunAudioLLM/CosyVoice/archive/074ca6dc9e80a2f424f1f74b48bdd7d3fea531cc.zip" MATCHA_ZIP = "https://github.com/shivammehta25/Matcha-TTS/archive/dd9105b34bf2be2230f4aa1e4769fb586a3c824e.zip" UNINSTALL = "--m0-uninstall" # name -> (python version, list of install steps; each step is the argument list for `uv pip install`) SPECS: dict[str, tuple[str, list[list[str]]]] = { "base": ("3.12", [["pillow==12.2.0"]]), "stt": ("3.12", [["faster-whisper==1.2.1", "ctranslate2==4.8.1", "nvidia-cublas-cu12==12.9.2.10", "nvidia-cudnn-cu12==9.24.0.43"]]), # MeloTTS pulls mecab-python3 (package dir "MeCab", Japanese only) and the # Korean g2p needs python-mecab-ko (package dir "mecab"). On Windows the two # folder names are the same folder, so mecab-python3 is removed first and # tts_melo.py stubs the Japanese tagger. "melo": ("3.11", [TORCH_CU128, [MELO_URL], [UNINSTALL, "mecab-python3"], ["python-mecab-ko==1.3.7", "python-mecab-ko-dic==2.1.1.post2"]]), "cosy": ("3.10", [TORCH_CU128, # openai-whisper is an sdist whose setup.py imports pkg_resources. ["setuptools<81", "wheel"], ["--no-build-isolation", "--no-deps", "openai-whisper==20231117"], ["numba", "tiktoken", "more-itertools", "tqdm"], ["conformer==0.3.2", "diffusers==0.29.0", "HyperPyYAML==1.2.3", "inflect==7.3.1", "librosa==0.10.2", "modelscope==1.20.0", "numpy==1.26.4", "omegaconf==2.3.0", "onnxruntime==1.18.0", "soundfile==0.12.1", "transformers==4.51.3", "x-transformers==2.11.24", "wetext==0.0.4", "hydra-core==1.3.2", "lightning==2.2.4", "matplotlib==3.7.5", "huggingface_hub==0.36.2", "rich==13.7.1", "pyarrow==18.1.0", "gdown==5.1.0", "wget==3.2", "pyworld==0.3.4"]]), } def venv_python(venv: Path) -> Path: return venv / ("Scripts/python.exe" if IS_WINDOWS else "bin/python") def find_uv() -> str: uv = os.environ.get("M0_UV") or shutil.which("uv") if not uv: raise RuntimeError("uv not found (run_m0.ps1 installs it)") return uv def ensure_env(name: str, work: Path, log: Path) -> Path: """Create the venv for ``name`` once; returns its python. Raises on failure.""" py_ver, steps = SPECS[name] venv = work / "venvs" / name py = venv_python(venv) done = venv / ".m0-ready" if done.exists() and py.exists(): return py uv = find_uv() say(f" [{name}] Python {py_ver} 가상환경 준비 중 (처음 한 번만, 몇 분 걸립니다)") if run_logged([uv, "venv", "--clear", "--python", py_ver, str(venv)], log, timeout=1800) != 0: raise RuntimeError(f"uv venv failed for {name} (see {log.name})") pinned = False for i, args in enumerate(steps, 1): say(f" [{name}] 패키지 설치 {i}/{len(steps)}") extra: list[str] = [] if pinned: pin = venv / "torch-pin.txt" pin.write_text(TORCH_PIN, encoding="utf-8") extra = ["-c", str(pin), "--extra-index-url", CU128_INDEX, "--index-strategy", "unsafe-best-match"] if args[0] == UNINSTALL: run_logged([uv, "pip", "uninstall", "--python", str(py), *args[1:]], log, timeout=600) _remove_package_dirs(venv, {a.lower() for a in args[1:]}) continue if run_logged([uv, "pip", "install", "--python", str(py), *args, *extra], log, timeout=3600) != 0: raise RuntimeError(f"package install step {i} failed for {name} (see {log.name})") pinned = pinned or args is TORCH_CU128 if name == "melo": # The full `unidic` package ships without its dictionary; with it removed # mecab-python3 falls back to unidic_lite (same fix as docker/setup-melo.sh). run_logged([uv, "pip", "uninstall", "--python", str(py), "unidic"], log, timeout=600) done.write_text("ok", encoding="utf-8") return py def _remove_package_dirs(venv: Path, dists: set[str]) -> None: """Remove folders an uninstall can leave behind (a case-insensitive clash on Windows).""" names = {"mecab-python3": "mecab"} targets = {names.get(d, d) for d in dists} for sp in list(venv.glob("Lib/site-packages")) + list(venv.glob("lib/python*/site-packages")): for entry in sp.iterdir(): if entry.is_dir() and entry.name.lower() in targets: shutil.rmtree(entry, ignore_errors=True) def _unzip_github(url: str, dest: Path, log: Path) -> None: say(f" 소스 받는 중: {url.split('/')[4]}") with urllib.request.urlopen(url, timeout=600) as resp: data = resp.read() with zipfile.ZipFile(io.BytesIO(data)) as z: top = z.namelist()[0].split("/")[0] tmp = dest.parent / (dest.name + ".tmp") shutil.rmtree(tmp, ignore_errors=True) z.extractall(tmp) shutil.rmtree(dest, ignore_errors=True) (tmp / top).rename(dest) shutil.rmtree(tmp, ignore_errors=True) with open(log, "a", encoding="utf-8") as f: f.write(f"unpacked {url} -> {dest.name}\n") def ensure_cosy_source(work: Path, log: Path) -> Path: src = work / "src" / "CosyVoice" if (src / "cosyvoice").is_dir() and (src / "third_party" / "Matcha-TTS" / "matcha").is_dir(): return src src.parent.mkdir(parents=True, exist_ok=True) _unzip_github(COSY_ZIP, src, log) _unzip_github(MATCHA_ZIP, src / "third_party" / "Matcha-TTS", log) return src