diff --git a/.env.example b/.env.example index 496cae5..a488413 100644 --- a/.env.example +++ b/.env.example @@ -34,23 +34,18 @@ WHISPER_DEVICE=cuda WHISPER_COMPUTE_TYPE=float16 # Optional explicit Piper voice model (.onnx). If empty, the jarvis default is used. TTS_PIPER_MODEL_PATH= -# TTS engine: "xtts" (default) uses the Coqui XTTS-v2 natural Korean voice -# served by the warm xtts-worker. Set to "piper" to use the English Piper voice -# directly. (MeloTTS was removed; "melo" only works with an out-of-band worker.) -TTS_ENGINE=xtts -# XTTS-v2 voice settings. Speaker is any built-in studio voice; "Ana Florence" -# is a natural female voice. Language is the synthesis language (ko = Korean). -XTTS_SPEAKER=Ana Florence -XTTS_LANGUAGE=ko -XTTS_DEVICE=cuda -# Where the bridge reaches the in-container XTTS worker, and how long it waits -# for a synthesis (XTTS is slower than Melo: ~1-2s/sentence on GPU). -XTTS_WORKER_URL=http://127.0.0.1:8771 -XTTS_TIMEOUT=30 -# Neural-only by default: if XTTS synthesis fails the bridge returns no audio +# TTS engine: "melo" (default) uses the MeloTTS Korean voice served by the warm +# melo-worker (Korean speaker, speed 1.5). Set to "piper" to use Piper directly. +TTS_ENGINE=melo +# Melo-only by default: if MeloTTS synthesis fails the bridge returns no audio # rather than speaking Korean through the English Piper voice (which mangles it). # Set to 1 only if you explicitly want the Piper fallback. -XTTS_FALLBACK_PIPER=0 +MELO_FALLBACK_PIPER=0 +# Where the bridge reaches the in-container MeloTTS worker, and how long it +# waits for a synthesis. Speaking rate is set on the worker via MELO_SPEED. +MELO_WORKER_URL=http://127.0.0.1:8770 +MELO_TIMEOUT=30 +MELO_SPEED=1.5 # --------------------------------------------------------------------------- # Jarvis brain (Ollama-backed). In Docker these populate the rendered @@ -231,7 +226,7 @@ COMPOSE_FILE=docker-compose.yml:docker-compose.gpu-linux.yml # OLLAMA_CHAT_MODEL=qwen2.5:7b # quality (needs ~5GB VRAM + whisper small) # OLLAMA_CHAT_MODEL=qwen2.5:3b # speed (fits easily, faster on 8GB GPUs) # WHISPER_MODEL=small # small frees VRAM for a bigger LLM; medium=more accurate -# XTTS_DEVICE=cuda # cpu if no GPU on the bot host (XTTS is slow on CPU) +# MELO_DEVICE=cuda # cpu if no GPU on the bot host # --- Settings web UI (http://localhost:8765/settings on the bot host) --- # To reach it, expose the bridge to the host loopback: diff --git a/Dockerfile b/Dockerfile index 628fc10..0f5c837 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,19 +65,18 @@ RUN ls -d /opt/venv/lib/python*/site-packages/nvidia/cublas/lib \ > /etc/ld.so.conf.d/nvidia-cu12.conf 2>/dev/null \ && /sbin/ldconfig || true -# --- Korean voice: Coqui XTTS-v2 (separate /opt/xtts py3.11 venv; see -# setup-xtts.sh). Natural female Korean ("Ana Florence"); replaces MeloTTS. -# Heavy layer (torch cu128 + Coqui TTS + the baked XTTS-v2 model); placed -# before the app COPY so it stays cached across source-only changes. --- -COPY docker/setup-xtts.sh /app/docker/setup-xtts.sh +# --- MeloTTS Korean voice (separate /opt/melo py3.11 venv; see setup-melo.sh). +# Heavy layer (torch CPU + transformers + MeCab); placed before the app +# COPY so it stays cached across source-only changes. --- +COPY docker/setup-melo.sh /app/docker/setup-melo.sh # Strip CR before running: a Windows checkout (autocrlf) yields CRLF, which makes -# bash read `set -euxo pipefail\r` and abort with "set: pipefail: invalid option -# name". .gitattributes pins *.sh to LF, but this keeps the build working even on -# a not-yet-renormalised working tree. -RUN sed -i 's/\r$//' /app/docker/setup-xtts.sh && bash /app/docker/setup-xtts.sh +# bash read line 18 as `set -euxo pipefail\r` and abort with +# "set: pipefail: invalid option name". .gitattributes pins *.sh to LF, but this +# keeps the build working even on a not-yet-renormalised working tree. +RUN sed -i 's/\r$//' /app/docker/setup-melo.sh && bash /app/docker/setup-melo.sh # --- Human input + window management for the on-screen Chrome control tool. -# Placed AFTER the heavy TTS layer so it doesn't bust that cache. xdotool +# Placed AFTER the heavy melo layer so it doesn't bust that cache. xdotool # injects real X pointer/keyboard events (visible cursor, char-by-char # typing) into the broadcast; wmctrl lists/moves windows. --- RUN apt-get update && apt-get install -y --no-install-recommends \ diff --git a/README.md b/README.md index edaf816..f57095a 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ docker compose -f docker-compose.yml -f docker-compose.gpu-linux.yml up -d --bui docker compose -f docker-compose.yml -f docker-compose.gpu-windows.yml up -d --build # ── GPU 없이 (CPU 전용 호스트) ── -# .env 에 WHISPER_DEVICE=cpu, XTTS_DEVICE=cpu 를 넣고 베이스만 사용 +# .env 에 WHISPER_DEVICE=cpu, MELO_DEVICE=cpu 를 넣고 베이스만 사용 docker compose up -d --build ``` @@ -113,7 +113,7 @@ docker compose up -d # 유저봇이 로그인해 지정 음성채널에 ### GPU 가속 (OS별) -LLM(Ollama), Whisper STT, XTTS-v2 TTS가 GPU에서 돕니다(env 기본 `WHISPER_DEVICE=cuda`, `XTTS_DEVICE=cuda`). NVIDIA Blackwell(sm_120, 예: RTX 5050/5070Ti)에서 검증: 컨테이너 내 torch cu128 CUDA 동작, Ollama GPU 오프로드, faster-whisper float16, XTTS-v2 GPU 합성 모두 확인. +LLM(Ollama), Whisper STT, MeloTTS가 GPU에서 돕니다(env 기본 `WHISPER_DEVICE=cuda`, `MELO_DEVICE=cuda`). NVIDIA Blackwell(sm_120, 예: RTX 5050/5070Ti)에서 검증: 컨테이너 내 torch cu128 CUDA 동작, Ollama GPU 오프로드, faster-whisper float16, MeloTTS GPU 합성 모두 확인. GPU는 위 "실행 — Docker"의 OS별 override 파일로 켜집니다. 호스트 사전 준비는 OS마다 다릅니다: @@ -137,7 +137,7 @@ docker run --rm --device nvidia.com/gpu=all ubuntu nvidia-smi -L # GPU 보이 **공통:** - 모델: 베이스 compose 기본은 `qwen2.5:3b`(8GB VRAM에서 도구호출 안정적). 더 무겁게(`qwen2.5:7b`, `qwen3:8b` 등) 쓰려면 `.env`의 `OLLAMA_CHAT_MODEL` 변경. -- **GPU가 없거나 인식 실패 시 자동으로 CPU 폴백**(Whisper)하므로 안전합니다. 명시적으로 CPU만 쓰려면 override 파일 없이 베이스만 올리고 `.env`에 `WHISPER_DEVICE=cpu`, `XTTS_DEVICE=cpu`를 두세요. +- **GPU가 없거나 인식 실패 시 자동으로 CPU 폴백**(Whisper)하므로 안전합니다. 명시적으로 CPU만 쓰려면 override 파일 없이 베이스만 올리고 `.env`에 `WHISPER_DEVICE=cpu`, `MELO_DEVICE=cpu`를 두세요. - 데이터(메모리 DB), Whisper 캐시, Piper 음성은 named volume에 영속됩니다. - 셀프봇 영상 송출 의존성은 이미지에 기본 포함하지 않습니다. 쓰려면 컨테이너에서 `cd /app/bot && bun add discord.js-selfbot-v13 @dank074/discord-video-stream` 후 재시작(또는 Dockerfile에 추가). @@ -243,7 +243,7 @@ cd bot; bun run register; bun run start # 창 2: (일반 봇이면) 슬래시 - `BRIDGE_URL` — 봇이 호출할 브릿지 주소 (기본 `http://127.0.0.1:8765`) - `STREAM_BACKEND`, `DISCORD_SELFBOT_TOKEN`, `NOVNC_URL` — 화면 송출 - `VNC_DISPLAY=:1`, `VNC_RESOLUTION`, `VNC_FRAMERATE`, `VNC_BITRATE_KBPS` — 캡처 -- `WHISPER_DEVICE/COMPUTE_TYPE`, `XTTS_DEVICE` — GPU 호스트면 `cuda`/`float16`, CPU 전용이면 `cpu`(GPU 자체는 OS별 override compose 파일로 켬) +- `WHISPER_DEVICE/COMPUTE_TYPE`, `MELO_DEVICE` — GPU 호스트면 `cuda`/`float16`, CPU 전용이면 `cpu`(GPU 자체는 OS별 override compose 파일로 켬) - `OLLAMA_CHAT_MODEL` — 두뇌 LLM (기본 `qwen2.5:3b`) - `COMPOSE_FILE` — OS별 GPU override를 매번 `-f`로 안 치고 자동 적용 (위 "실행 — Docker" 참고) - `output_language` — 출력 언어 고정(비우면 사용자 언어). 설정 웹 UI(`/settings`)에서 바꾸면 env 기본값보다 우선하며 컨테이너 재생성 후에도 유지됩니다. diff --git a/bridge/xtts_worker.py b/bridge/melo_worker.py similarity index 53% rename from bridge/xtts_worker.py rename to bridge/melo_worker.py index 9352745..d1f57b7 100644 --- a/bridge/xtts_worker.py +++ b/bridge/melo_worker.py @@ -1,30 +1,25 @@ """ -XTTS worker -=========== +MeloTTS worker +============== -A tiny HTTP service that keeps a Coqui XTTS-v2 voice warm and synthesises -speech on demand. It mirrors ``melo_worker.py`` (same ``/synth`` + ``/health`` -contract, same PCM16 WAV output) so the bridge can talk to either worker the -same way. +A tiny, dependency-light HTTP service that keeps a MeloTTS voice warm and +synthesises speech on demand. It runs in its OWN Python venv (``/opt/melo`` in +the container) so the heavy MeloTTS/torch/transformers stack stays isolated +from the slim brain-bridge venv (which pins ``numpy<2`` for faster-whisper). -XTTS-v2 is a natural, multilingual neural voice. The default speaker is the -built-in female studio voice "Ana Florence" speaking Korean — the voice this -deployment uses in place of MeloTTS. No reference WAV is needed for the -built-in studio speakers. - -It runs in its OWN Python venv (``/opt/xtts`` in the container) so the heavy -Coqui TTS / torch stack stays isolated from the slim brain-bridge venv. +The bridge's ``synthesize()`` POSTs ``{"text": "..."}`` here and gets back a +16-bit PCM WAV. The MeloTTS model is loaded once at startup and reused, so each +request only pays inference cost, not model-load cost. Config (env): - XTTS_WORKER_HOST bind host (default 127.0.0.1) - XTTS_WORKER_PORT bind port (default 8771) - XTTS_MODEL Coqui model id (default tts_models/multilingual/multi-dataset/xtts_v2) - XTTS_SPEAKER built-in speaker (default "Ana Florence") - XTTS_LANGUAGE synthesis language (default ko) - XTTS_DEVICE torch device (default cpu; compose sets cuda) + MELO_WORKER_HOST bind host (default 127.0.0.1) + MELO_WORKER_PORT bind port (default 8770) + MELO_LANGUAGE MeloTTS language (default KR) + MELO_SPEED speaking rate (default 1.5 -> the approved "150") + MELO_DEVICE torch device (default cpu) Run: - /opt/xtts/bin/python -m bridge.xtts_worker + /opt/melo/bin/python -m bridge.melo_worker """ from __future__ import annotations @@ -38,72 +33,94 @@ import threading import wave from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -# XTTS-v2 is gated behind a one-time license prompt; agreeing here keeps the -# load non-interactive in a container. XTTS-v2 is non-commercial (CPML). -os.environ.setdefault("COQUI_TOS_AGREED", "1") +HOST = os.environ.get("MELO_WORKER_HOST", "127.0.0.1") +PORT = int(os.environ.get("MELO_WORKER_PORT", "8770")) +LANGUAGE = os.environ.get("MELO_LANGUAGE", "KR") -HOST = os.environ.get("XTTS_WORKER_HOST", "127.0.0.1") -PORT = int(os.environ.get("XTTS_WORKER_PORT", "8771")) -MODEL = os.environ.get("XTTS_MODEL", "tts_models/multilingual/multi-dataset/xtts_v2") -SPEAKER = os.environ.get("XTTS_SPEAKER", "Ana Florence") -LANGUAGE = os.environ.get("XTTS_LANGUAGE", "ko") -DEVICE = os.environ.get("XTTS_DEVICE", "cpu") -# Model is loaded once, guarded by a lock because TTS inference is not -# guaranteed thread-safe. +def _resolve_speed() -> float: + """Speaking rate: the settings-UI value (runtime config JSON) wins, else the + MELO_SPEED env, else 1.5. Read at startup; the settings UI restarts this + worker on apply so a new value takes effect.""" + try: + cp = os.environ.get("JARVIS_CONFIG_PATH", "/app/config/jarvis.json") + v = json.loads(open(cp, encoding="utf-8").read()).get("melo_speed") + if v is not None: + return float(v) + except Exception: + pass + try: + return float(os.environ.get("MELO_SPEED", "1.5")) + except ValueError: + return 1.5 + + +SPEED = _resolve_speed() +DEVICE = os.environ.get("MELO_DEVICE", "cpu") + +# Model + speaker id are loaded once, guarded by a lock because MeloTTS +# inference is not guaranteed thread-safe. _model = None +_speaker_id = None _model_lock = threading.Lock() _load_error: str | None = None def _ensure_model() -> None: - global _model, _load_error + global _model, _speaker_id, _load_error if _model is not None or _load_error is not None: return with _model_lock: if _model is not None or _load_error is not None: return try: - from TTS.api import TTS # type: ignore + from melo.api import TTS # type: ignore - model = TTS(MODEL).to(DEVICE) + model = TTS(language=LANGUAGE, device=DEVICE) + # spk2id is a melo HParams object (dict-like, supports __getitem__, + # __contains__, keys) but NOT .get(). The KR model exposes a single + # 'KR' speaker; fall back to the first id for other languages. + spk_map = model.hps.data.spk2id + keys = list(spk_map.keys()) + speaker_id = spk_map[LANGUAGE] if LANGUAGE in spk_map else spk_map[keys[0]] _model = model - # Warm once: the first GPU synth pays a one-off kernel-init cost - # that would otherwise land on the user's first reply. + _speaker_id = speaker_id + # Warm the GPU once at load: the first CUDA synth pays a one-off + # kernel-init cost (~5s) that would otherwise land on the user's + # first reply. A throwaway synth here moves it to startup. No-op + # cost on CPU. try: with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as _wt: _wp = _wt.name - model.tts_to_file( - text="워밍업", speaker=SPEAKER, language=LANGUAGE, file_path=_wp - ) + model.tts_to_file("워밍업", speaker_id, _wp, speed=SPEED) try: os.unlink(_wp) except OSError: pass except Exception as _we: # pragma: no cover - print(f"[xtts-worker] warmup synth skipped: {_we}", flush=True) + print(f"[melo-worker] warmup synth skipped: {_we}", flush=True) print( - f"[xtts-worker] ready (model={MODEL} speaker={SPEAKER!r} " - f"language={LANGUAGE} device={DEVICE})", + f"[melo-worker] ready (lang={LANGUAGE} speed={SPEED} " + f"device={DEVICE} speakers={list(spk_map.keys())})", flush=True, ) except Exception as e: # pragma: no cover - depends on local model files _load_error = f"{type(e).__name__}: {e}" - print(f"[xtts-worker] model load FAILED: {_load_error}", flush=True) + print(f"[melo-worker] model load FAILED: {_load_error}", flush=True) def _synthesize(text: str) -> bytes: """Synthesise ``text`` to a 16-bit PCM WAV (bytes).""" _ensure_model() if _model is None: - raise RuntimeError(_load_error or "xtts model unavailable") + raise RuntimeError(_load_error or "melo model unavailable") + # MeloTTS writes to a file via soundfile; render to a container-disk temp + # file (NOT tmpfs), read it back, then drop it. with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp: tmp_path = tmp.name try: with _model_lock: - _model.tts_to_file( - text=text, speaker=SPEAKER, language=LANGUAGE, file_path=tmp_path - ) + _model.tts_to_file(text, _speaker_id, tmp_path, speed=SPEED) with open(tmp_path, "rb") as f: raw = f.read() finally: @@ -115,15 +132,16 @@ def _synthesize(text: str) -> bytes: def _ensure_pcm16_wav(raw: bytes) -> bytes: - """Guarantee a 16-bit PCM WAV. Coqui writes float/other WAVs; the Discord - playback path tolerates both, but we normalise to PCM16 so the contract - matches the previous Melo/Piper output (mono, file's own sample rate).""" + """Guarantee a 16-bit PCM WAV. MeloTTS/soundfile usually emit float WAVs; + the Discord playback path (ffmpeg) tolerates both, but we normalise to + PCM16 so the contract matches the previous Piper output.""" try: with wave.open(io.BytesIO(raw), "rb") as wf: if wf.getsampwidth() == 2: return raw # already PCM16 except wave.Error: pass + # Non-PCM16 (e.g. float) — convert with soundfile if available. try: import numpy as np import soundfile as sf @@ -141,7 +159,7 @@ def _ensure_pcm16_wav(raw: bytes) -> bytes: wf.writeframes(pcm) return buf.getvalue() except Exception: - return raw # last resort: hand back whatever XTTS produced + return raw # last resort: hand back whatever MeloTTS produced class _Handler(BaseHTTPRequestHandler): @@ -194,7 +212,7 @@ def main() -> int: # Warm the model at startup so the first Discord turn isn't slow. _ensure_model() server = ThreadingHTTPServer((HOST, PORT), _Handler) - print(f"[xtts-worker] listening on http://{HOST}:{PORT}", flush=True) + print(f"[melo-worker] listening on http://{HOST}:{PORT}", flush=True) try: server.serve_forever() except KeyboardInterrupt: diff --git a/bridge/server.py b/bridge/server.py index 5521e32..285c868 100644 --- a/bridge/server.py +++ b/bridge/server.py @@ -87,13 +87,12 @@ VAD_MIN_SPEECH_MS = int(os.environ.get("VAD_MIN_SPEECH_MS", "200")) # Korean phrase decoded as Chinese) and shaves a little latency. Empty = auto. STT_LANGUAGE = os.environ.get("STT_LANGUAGE", "ko").strip() or None -# TTS engine: "xtts" (Coqui XTTS-v2 natural Korean voice, the warm worker) is -# the primary voice; Piper is kept as a fallback only if explicitly enabled. Set -# TTS_ENGINE=piper to disable the neural Korean voice entirely. "melo" is still -# accepted for backward compatibility but is no longer built into the image. +# TTS engine: "melo" (MeloTTS Korean speaker, the warm worker) is the primary +# voice; Piper is kept as a fallback if the worker is unreachable. Set +# TTS_ENGINE=piper to disable MeloTTS entirely. def _tts_engine_setting() -> str: """TTS engine: settings-UI value (runtime config JSON) wins, else env, else - xtts. Read at startup; the settings UI restarts the bridge on apply.""" + melo. Read at startup; the settings UI restarts the bridge on apply.""" try: _cp = os.environ.get("JARVIS_CONFIG_PATH", "/app/config/jarvis.json") _v = json.loads(open(_cp, encoding="utf-8").read()).get("tts_engine") @@ -101,29 +100,17 @@ def _tts_engine_setting() -> str: return str(_v).strip().lower() except Exception: pass - return os.environ.get("TTS_ENGINE", "xtts").strip().lower() + return os.environ.get("TTS_ENGINE", "melo").strip().lower() TTS_ENGINE = _tts_engine_setting() -# Coqui XTTS-v2 worker (the natural Korean voice). -XTTS_WORKER_URL = os.environ.get("XTTS_WORKER_URL", "http://127.0.0.1:8771") -XTTS_TIMEOUT = float(os.environ.get("XTTS_TIMEOUT", "30")) -# Legacy MeloTTS worker (no longer built into the image; kept for back-compat -# if someone runs an old worker out-of-band). MELO_WORKER_URL = os.environ.get("MELO_WORKER_URL", "http://127.0.0.1:8770") MELO_TIMEOUT = float(os.environ.get("MELO_TIMEOUT", "30")) -# Do NOT silently fall back to the English Piper voice on a neural-voice failure: -# speaking Korean text through an English voice produces mangled audio. Default -# is neural-only (return no audio on failure); set XTTS_FALLBACK_PIPER=1 (or the -# legacy MELO_FALLBACK_PIPER=1) to opt into the Piper fallback. -def _truthy_env(*names: str) -> bool: - for _n in names: - if os.environ.get(_n, "").strip().lower() in ("1", "true", "yes", "on"): - return True - return False - - -NEURAL_FALLBACK_PIPER = _truthy_env("XTTS_FALLBACK_PIPER", "MELO_FALLBACK_PIPER") +# When MeloTTS is the engine, do NOT silently fall back to the English Piper +# voice on failure: speaking Korean text through an English voice produces +# mangled audio. Default is melo-only (return no audio on failure); set +# MELO_FALLBACK_PIPER=1 to opt into the Piper fallback. +MELO_FALLBACK_PIPER = os.environ.get("MELO_FALLBACK_PIPER", "0") in ("1", "true", "True", "yes", "on") # --------------------------------------------------------------------------- # Lazy singletons. The first request pays the model-load cost; afterwards the @@ -315,38 +302,27 @@ def _coerce_bool(value) -> Optional[bool]: return str(value).strip().lower() in ("1", "true", "yes", "on") -def _worker_synthesize(name: str, url: str, timeout: float, text: str) -> Optional[bytes]: - """POST text to a warm TTS worker's /synth and return its WAV bytes, or None - on any failure so the caller can decide whether to fall back.""" +def _melo_synthesize(text: str) -> Optional[bytes]: + """Synthesise via the warm MeloTTS worker (separate /opt/melo venv, Korean + speaker @ speed 1.5). Returns a 16-bit PCM WAV, or None on any failure so + the caller can fall back to Piper.""" import urllib.request try: req = urllib.request.Request( - f"{url}/synth", + f"{MELO_WORKER_URL}/synth", data=json.dumps({"text": text}).encode("utf-8"), headers={"Content-Type": "application/json"}, ) - with urllib.request.urlopen(req, timeout=timeout) as resp: + with urllib.request.urlopen(req, timeout=MELO_TIMEOUT) as resp: if resp.status == 200: return resp.read() - print(f"[bridge] {name} worker HTTP {resp.status}", flush=True) + print(f"[bridge] melo worker HTTP {resp.status}", flush=True) except Exception as e: # pragma: no cover - worker may be down - print(f"[bridge] {name} worker unreachable: {e}", flush=True) + print(f"[bridge] melo worker unreachable: {e}", flush=True) return None -def _xtts_synthesize(text: str) -> Optional[bytes]: - """Synthesise via the warm Coqui XTTS-v2 worker (separate /opt/xtts venv, - natural female Korean). Returns a 16-bit PCM WAV, or None on failure.""" - return _worker_synthesize("xtts", XTTS_WORKER_URL, XTTS_TIMEOUT, text) - - -def _melo_synthesize(text: str) -> Optional[bytes]: - """Legacy: synthesise via a MeloTTS worker if one is running out-of-band. - Returns a 16-bit PCM WAV, or None on any failure.""" - return _worker_synthesize("melo", MELO_WORKER_URL, MELO_TIMEOUT, text) - - def _piper_synthesize(text: str) -> Optional[bytes]: """Fallback: synthesise with Piper (English voice). Returns WAV bytes.""" _ensure_piper() @@ -373,12 +349,11 @@ def _tts_ready() -> bool: """ if not TTS_ENABLED: return True - _worker_health = {"xtts": XTTS_WORKER_URL, "melo": MELO_WORKER_URL}.get(TTS_ENGINE) - if _worker_health: + if TTS_ENGINE == "melo": import urllib.request try: - with urllib.request.urlopen(f"{_worker_health}/health", timeout=2) as resp: + with urllib.request.urlopen(f"{MELO_WORKER_URL}/health", timeout=2) as resp: return resp.status == 200 except Exception: return False @@ -386,24 +361,20 @@ def _tts_ready() -> bool: def synthesize(text: str) -> Optional[bytes]: - """Synthesize text to a 16-bit PCM WAV. The primary voice is Coqui XTTS-v2 - (natural female Korean) served by the warm xtts worker; Piper is used only - when explicitly enabled as a fallback. Returns None if TTS is off.""" + """Synthesize text to a 16-bit PCM WAV. The primary voice is MeloTTS + (Korean speaker, speed 1.5) served by the warm melo worker; Piper is a + fallback if the worker is unavailable. Returns None if TTS is off.""" if not TTS_ENABLED or not text.strip(): return None - _neural = {"xtts": _xtts_synthesize, "melo": _melo_synthesize}.get(TTS_ENGINE) - if _neural is not None: - audio = _neural(text) + if TTS_ENGINE == "melo": + audio = _melo_synthesize(text) if audio: return audio - if not NEURAL_FALLBACK_PIPER: - # Neural-only: better silent than mangled English for Korean text. - print( - f"[bridge] {TTS_ENGINE} synth failed; no audio (Piper fallback disabled)", - flush=True, - ) + if not MELO_FALLBACK_PIPER: + # Melo-only: better silent than mangled English for Korean text. + print("[bridge] melo synth failed; no audio (Piper fallback disabled)", flush=True) return None - print(f"[bridge] {TTS_ENGINE} synth failed; falling back to Piper", flush=True) + print("[bridge] melo synth failed; falling back to Piper", flush=True) return _piper_synthesize(text) diff --git a/bridge/settings_web.py b/bridge/settings_web.py index eeed206..bdb97af 100644 --- a/bridge/settings_web.py +++ b/bridge/settings_web.py @@ -22,7 +22,8 @@ from typing import Any, Dict FIELDS = [ ("ollama_chat_model", "LLM 모델", "model"), ("whisper_model", "STT(Whisper) 모델", "select:tiny,base,small,medium,large,large-v3"), - ("tts_engine", "TTS 엔진", "select:xtts,piper"), + ("tts_engine", "TTS 엔진", "select:melo,piper"), + ("melo_speed", "TTS 속도 (MeloTTS)", "number:0.5:2.5:0.1"), ("output_language", "출력 언어 (비우면 사용자 언어)", "text"), ("llm_thinking_enabled", "LLM 사고(thinking) 모드", "bool"), ("agentic_max_turns", "에이전트 최대 반복", "number:1:12:1"), @@ -53,7 +54,9 @@ def _current() -> Dict[str, Any]: cfg = _read_config() out: Dict[str, Any] = {} for k in _KEYS: - if k == "output_language": + if k == "melo_speed": + out[k] = cfg.get("melo_speed", os.environ.get("MELO_SPEED", "1.5")) + elif k == "output_language": out[k] = cfg.get("output_language", os.environ.get("OUTPUT_LANGUAGE", "")) else: out[k] = cfg.get(k, "") @@ -75,7 +78,12 @@ def _coerce(updates: Dict[str, Any]) -> Dict[str, Any]: for k, v in updates.items(): if k not in _KEYS: continue - if k == "agentic_max_turns": + if k == "melo_speed": + try: + v = float(v) + except (TypeError, ValueError): + continue + elif k == "agentic_max_turns": try: v = int(v) except (TypeError, ValueError): @@ -106,12 +114,12 @@ def _save(updates: Dict[str, Any]) -> None: def _apply() -> str: - # Restart the TTS worker + bridge AFTER this response is sent. Detached (new - # session) so the bridge being killed mid-restart doesn't drop the restart - # itself, and the HTTP client still receives this response. + # Restart melo + bridge AFTER this response is sent. Detached (new session) + # so the bridge being killed mid-restart doesn't drop the restart itself, + # and the HTTP client still receives this response. try: subprocess.Popen( - ["sh", "-c", "sleep 1; supervisorctl restart xtts-worker bridge"], + ["sh", "-c", "sleep 1; supervisorctl restart melo-worker bridge"], start_new_session=True, ) return "1초 후 브리지/TTS 워커가 재시작되어 반영됩니다." diff --git a/docker-compose.yml b/docker-compose.yml index d06ad55..9f63909 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -66,15 +66,13 @@ services: WHISPER_MODEL: ${WHISPER_MODEL:-medium} WHISPER_DEVICE: ${WHISPER_DEVICE:-cuda} WHISPER_COMPUTE_TYPE: ${WHISPER_COMPUTE_TYPE:-float16} - # Coqui XTTS-v2 (natural female Korean voice, replaces MeloTTS) on the GPU - # (cu128 torch baked by docker/setup-xtts.sh). Set here WITH DEFAULTS so - # supervisord's %(ENV_XTTS_*)s passthrough always resolves and an .env - # override actually reaches the xtts-worker. - XTTS_DEVICE: ${XTTS_DEVICE:-cuda} - # Built-in studio speaker (female). Other options include "Daisy Studious", - # "Sofia Hellen", "Alma María", etc. — any XTTS-v2 studio speaker name. - XTTS_SPEAKER: ${XTTS_SPEAKER:-Ana Florence} - XTTS_LANGUAGE: ${XTTS_LANGUAGE:-ko} + # MeloTTS on the GPU (cu128 torch baked by docker/setup-melo.sh). CPU synth + # serialised under load and pushed TTS to 7-8s; GPU does ~0.3s/sentence. + MELO_DEVICE: ${MELO_DEVICE:-cuda} + # Speaking rate for MeloTTS. Set here (with a default) so supervisord's + # %(ENV_MELO_SPEED)s passthrough always resolves and an .env override + # actually reaches the melo-worker. Lower it (e.g. 1.1) for a calmer pace. + MELO_SPEED: ${MELO_SPEED:-1.5} # Optional single-language lock for replies (empty = user's own language). OUTPUT_LANGUAGE: ${OUTPUT_LANGUAGE:-ko} # Drop the pre-loop planner LLM call to cut voice-reply latency on small diff --git a/docker/setup-melo.sh b/docker/setup-melo.sh new file mode 100755 index 0000000..5202bd7 --- /dev/null +++ b/docker/setup-melo.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# ============================================================================ +# Install a dedicated MeloTTS (Korean voice) venv at /opt/melo. +# +# Why a SEPARATE venv (not the brain-bridge /opt/venv): +# - MeloTTS pins old deps (transformers 4.27.4 / tokenizers 0.13.3 / fugashi) +# whose binary wheels exist only for cp311, so we use python3.11 here even +# though the image's default interpreter is 3.12. +# - It isolates the heavy torch/transformers stack from the slim bridge env, +# which pins numpy<2 for faster-whisper. +# +# torch is the CUDA (cu128) build so MeloTTS runs on the GPU alongside Ollama + +# Whisper. CPU synth serialised under concurrent load (whisper STT + bot) and +# blew TTS up to 7-8s per reply; on the GPU a sentence synthesises in ~0.3s. +# cu128 is the Blackwell (sm_120) wheel verified on this host's RTX 5050. +# The worker selects the device via MELO_DEVICE=cuda (compose). +# ============================================================================ +set -euxo pipefail + +export DEBIAN_FRONTEND=noninteractive + +apt-get update +# Build deps for fugashi / mecab-python3 + a system MeCab dict, plus python3.11. +apt-get install -y --no-install-recommends \ + software-properties-common build-essential pkg-config swig \ + libmecab-dev mecab mecab-ipadic-utf8 +add-apt-repository -y ppa:deadsnakes/ppa +apt-get update +apt-get install -y --no-install-recommends python3.11 python3.11-venv python3.11-dev +rm -rf /var/lib/apt/lists/* + +python3.11 -m venv /opt/melo +/opt/melo/bin/pip install --no-cache-dir --upgrade pip wheel setuptools + +# CUDA (cu128) torch first, so MeloTTS's unpinned `torch` dep is already +# satisfied with the GPU build. Pinned to the Blackwell-verified versions +# (2.11.0+cu128) for reproducible rebuilds. +/opt/melo/bin/pip install --no-cache-dir torch==2.11.0+cu128 torchaudio==2.11.0+cu128 \ + --index-url https://download.pytorch.org/whl/cu128 + +# MeloTTS from GitHub. The PyPI sdist is broken (its setup.py reads a +# requirements.txt that is not shipped in the sdist), so install from the repo. +# Pinned to a commit (not refs/heads/main) so rebuilds are reproducible. +/opt/melo/bin/pip install --no-cache-dir \ + "https://github.com/myshell-ai/MeloTTS/archive/209145371cff8fc3bd60d7be902ea69cbdb7965a.tar.gz" + +# Korean g2p backend. MeloTTS otherwise tries to pip-install this on the first +# Korean request, which fails in a network-isolated container at runtime. +/opt/melo/bin/pip install --no-cache-dir python-mecab-ko python-mecab-ko-dic + +# Remove the full `unidic` package (its dictionary is never downloaded, only a +# stub) so mecab-python3 falls back to the bundled `unidic_lite` dict. Without +# this, importing melo's Japanese module fails with a missing-mecabrc error. +/opt/melo/bin/pip uninstall -y unidic || true + +# Pre-cache every model asset MeloTTS pulls at runtime, so the worker starts +# offline and the first Discord turn pays no download cost. Importing melo.api +# fetches the Japanese (tohoku-nlp/bert-base-japanese-v3) and Korean +# (kykim/bert-kor-base) BERT tokenizers plus nltk g2p data; loading the KR voice +# downloads the OpenVoice KR config+checkpoint, and a real synth pulls the +# Korean BERT weights. All of these go through huggingface_hub. +# +# CRITICAL: at runtime docker-compose mounts the `whisper_cache` named volume +# over /root/.cache/huggingface (for faster-whisper). That volume would SHADOW +# anything baked into the default HF cache, so we pin the melo worker to a +# DEDICATED, non-volume cache dir (/opt/melo-cache) here AND in supervisord, and +# warm it once. nltk_data (/root/nltk_data) is not volume-mounted so it stays. +export HF_HOME=/opt/melo-cache +mkdir -p "$HF_HOME" +MELO_LANGUAGE=KR HF_HOME=/opt/melo-cache /opt/melo/bin/python - <<'PY' +import tempfile +from melo.api import TTS + +model = TTS(language="KR", device="cpu") +out = tempfile.mktemp(suffix=".wav") +model.tts_to_file("초기화 워밍업입니다.", model.hps.data.spk2id["KR"], out, speed=1.5) +print("[setup-melo] warm-up KR synth OK ->", out) +PY + +echo "[setup-melo] MeloTTS venv ready at /opt/melo" diff --git a/docker/setup-xtts.sh b/docker/setup-xtts.sh deleted file mode 100644 index 1493cae..0000000 --- a/docker/setup-xtts.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -# ============================================================================ -# Install a dedicated Coqui XTTS-v2 (natural Korean voice) venv at /opt/xtts. -# -# Why a SEPARATE venv (not the brain-bridge /opt/venv or /opt/melo): -# - Coqui TTS pulls its own heavy torch/transformers stack; isolating it keeps -# the slim bridge env (numpy<2 for faster-whisper) untouched. -# - We use python3.11 (installed for the melo layer) because Coqui ships cp311 -# wheels and torch cu128 is available for it. -# -# torch is the CUDA (cu128) build so XTTS runs on the GPU alongside Ollama + -# Whisper. cu128 is the Blackwell (sm_120) wheel verified on this host. -# The worker selects the device via XTTS_DEVICE=cuda (compose). -# -# XTTS-v2 is non-commercial (Coqui Public Model License). COQUI_TOS_AGREED=1 -# accepts it non-interactively so the model can load in a headless container. -# ============================================================================ -set -euxo pipefail - -export DEBIAN_FRONTEND=noninteractive -export COQUI_TOS_AGREED=1 - -# Install python3.11 if not already present, so this layer is self-contained. -if ! command -v python3.11 >/dev/null 2>&1; then - apt-get update - apt-get install -y --no-install-recommends software-properties-common - add-apt-repository -y ppa:deadsnakes/ppa - apt-get update - apt-get install -y --no-install-recommends python3.11 python3.11-venv python3.11-dev - rm -rf /var/lib/apt/lists/* -fi - -python3.11 -m venv /opt/xtts -/opt/xtts/bin/pip install --no-cache-dir --upgrade pip wheel setuptools - -# CUDA (cu128) torch first so Coqui's `torch` dep is satisfied with the GPU -# build. Pinned to the Blackwell-verified versions for reproducible rebuilds. -/opt/xtts/bin/pip install --no-cache-dir torch==2.11.0+cu128 torchaudio==2.11.0+cu128 \ - --index-url https://download.pytorch.org/whl/cu128 - -# Coqui TTS (maintained fork; provides the `TTS` package and XTTS-v2). The -# [codec] extra pulls torchcodec, which torch >=2.9 requires for audio IO -# (without it the import fails with TORCHCODEC_IMPORT_ERROR). torchcodec also -# needs the system FFmpeg shared libs, which are present (ffmpeg apt package). -/opt/xtts/bin/pip install --no-cache-dir "coqui-tts[codec]" - -# Pin transformers to the 4.57+ / <5 range. coqui-tts requires >=4.57 but does -# NOT cap the upper bound, and transformers 5.x removed `isin_mps_friendly` -# (used by XTTS's tortoise layer), so an unpinned install pulls 5.x and the -# model import fails with "cannot import name 'isin_mps_friendly'". Pin <5. -/opt/xtts/bin/pip install --no-cache-dir "transformers>=4.57,<5" - -# Pre-bake the XTTS-v2 model so the worker starts offline and the first Discord -# turn pays no download cost. The model is cached under TTS_HOME; we pin a -# DEDICATED, non-volume dir (/opt/xtts-cache) AND set it in supervisord, because -# runtime volume mounts (whisper_cache over /root/.cache) must not shadow it. -export TTS_HOME=/opt/xtts-cache -mkdir -p "$TTS_HOME" -COQUI_TOS_AGREED=1 TTS_HOME=/opt/xtts-cache XTTS_SPEAKER="Ana Florence" \ - /opt/xtts/bin/python - <<'PY' -import os -os.environ["COQUI_TOS_AGREED"] = "1" -from TTS.api import TTS - -speaker = os.environ.get("XTTS_SPEAKER", "Ana Florence") -model = TTS("tts_models/multilingual/multi-dataset/xtts_v2") # downloads to TTS_HOME -out = "/tmp/xtts_warm.wav" -model.tts_to_file(text="초기화 워밍업입니다.", speaker=speaker, language="ko", file_path=out) -print("[setup-xtts] warm-up KR synth OK ->", out, "speaker:", speaker) -PY - -echo "[setup-xtts] Coqui XTTS-v2 venv ready at /opt/xtts (cache /opt/xtts-cache)" diff --git a/docker/supervisord.conf b/docker/supervisord.conf index 9acbc8a..c512e1f 100644 --- a/docker/supervisord.conf +++ b/docker/supervisord.conf @@ -49,22 +49,25 @@ stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 -[program:xtts-worker] -; Warm Coqui XTTS-v2 Korean voice (natural female "Ana Florence") in its own -; py3.11 venv. The bridge's synthesize() POSTs here; if this is down the bridge -; falls back to Piper (English) only when XTTS_FALLBACK_PIPER=1. -command=/app/docker/run-if-role.sh full,bot /opt/xtts/bin/python /app/bridge/xtts_worker.py +[program:melo-worker] +; Warm MeloTTS Korean voice (speed 1.5) in its own py3.11 venv. The bridge's +; synthesize() POSTs here; if this is down the bridge falls back to Piper. +command=/app/docker/run-if-role.sh full,bot /opt/melo/bin/python /app/bridge/melo_worker.py directory=/app -; TTS_HOME points at the dedicated, image-baked XTTS cache (warmed in -; setup-xtts.sh). The brain's whisper_cache volume is mounted over -; /root/.cache, so a dedicated non-volume cache dir avoids the baked model being -; shadowed and re-downloaded (which would fail if the host is offline). -; XTTS_DEVICE / XTTS_SPEAKER / XTTS_LANGUAGE inherit from the container env -; (compose sets them with defaults: cuda / "Ana Florence" / ko). supervisord -; interpolates %(ENV_x)s from its own environment, which is the container's — so -; these must always be set in the env (compose guarantees it) or this expansion -; fails at startup. COQUI_TOS_AGREED accepts the non-commercial XTTS license. -environment=XTTS_DEVICE="%(ENV_XTTS_DEVICE)s",XTTS_SPEAKER="%(ENV_XTTS_SPEAKER)s",XTTS_LANGUAGE="%(ENV_XTTS_LANGUAGE)s",XTTS_WORKER_HOST="127.0.0.1",XTTS_WORKER_PORT="8771",TTS_HOME="/opt/xtts-cache",COQUI_TOS_AGREED="1" +; HF_HOME points at the dedicated, image-baked melo cache (warmed in +; setup-melo.sh). The brain's whisper_cache volume is mounted over +; /root/.cache/huggingface, so without this the pre-cached BERT + KR checkpoint +; would be shadowed and re-downloaded (and would fail if the host is offline). +; HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE force pure-cache reads: the pinned old +; transformers/huggingface_hub otherwise retry the network on every load and +; error out instead of falling back to the (complete) baked cache. +; MELO_DEVICE and MELO_SPEED inherit from the container env (compose sets both +; with defaults: cuda / 1.5) so the worker runs MeloTTS on the GPU at the +; configured rate. supervisord interpolates %(ENV_x)s from its own environment, +; which is the container's — so MELO_SPEED must always be set in the env +; (compose guarantees it) or this expansion fails at startup. Hardcoding 1.5 +; here previously shadowed the .env value, so lowering MELO_SPEED had no effect. +environment=MELO_LANGUAGE="KR",MELO_SPEED="%(ENV_MELO_SPEED)s",MELO_DEVICE="%(ENV_MELO_DEVICE)s",MELO_WORKER_HOST="127.0.0.1",MELO_WORKER_PORT="8770",HF_HOME="/opt/melo-cache",HF_HUB_OFFLINE="1",TRANSFORMERS_OFFLINE="1" priority=280 autorestart=true stdout_logfile=/dev/stdout diff --git a/tests/test_system_prompt.py b/tests/test_system_prompt.py index d89e614..3134989 100644 --- a/tests/test_system_prompt.py +++ b/tests/test_system_prompt.py @@ -101,14 +101,6 @@ class TestReplyLanguageDirective: # user's own language, so no directive. assert reply_language_directive(None, "melo") is None - def test_xtts_is_multilingual(self): - # XTTS-v2 (the Korean voice) is not English-only: no lock → free, and a - # lock is honoured (not overridden to English). - assert reply_language_directive(None, "xtts") is None - directive = reply_language_directive("Korean", "xtts") - assert directive is not None and "Korean" in directive - assert directive != ENGLISH_ONLY_DIRECTIVE - def test_unknown_tts_defaults_to_english_only(self): # Preserves the original getattr(cfg, 'tts_engine', 'piper') default: # an unknown/missing engine is treated conservatively as English-only.