From b17961e9e38709a1004a5738ce340fda983b8006 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Fri, 12 Jun 2026 19:01:54 +0900 Subject: [PATCH] feat(tts): add MeloTTS Korean voice via warm worker with offline-baked cache Adds a dedicated MeloTTS Korean voice (speed 1.5) as the primary TTS engine, served by a long-lived in-container worker so each Discord turn pays only inference cost, not model-load cost. - bridge/melo_worker.py: tiny HTTP service in its own /opt/melo py3.11 venv, keeps the KR model warm, returns PCM16 WAV on POST /synth. - bridge/server.py: synthesize() routes to the melo worker first; Piper stays as an opt-in fallback (MELO_FALLBACK_PIPER, default off so Korean is never mangled through the English voice). /health reports tts_engine. - docker/setup-melo.sh: builds the isolated venv (pinned torch 2.12.0 / torchaudio 2.11.0 CPU, MeloTTS pinned to a commit for reproducible rebuilds), pre-fetches mecab-ko, and warms a dedicated HF cache (/opt/melo-cache) with a real KR synth so all BERT + KR checkpoint assets are baked into the image. - docker/supervisord.conf: runs melo-worker before the bridge with HF_HOME=/opt/melo-cache (the whisper_cache volume shadows the default HF cache) plus HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE so it reads the baked cache and never retries the network on load. - Dockerfile/.env.example: wire the melo build layer and config knobs. Verified: offline synth passes with --network none and the prod volume mounted; prod container recreated, all supervisord services up, bot logged in, and an end-to-end /tts call returns a 44.1kHz mono PCM16 WAV. Co-Authored-By: Claude Opus 4.7 --- .env.example | 12 +++ Dockerfile | 6 ++ bridge/melo_worker.py | 191 ++++++++++++++++++++++++++++++++++++++++ bridge/server.py | 59 ++++++++++++- docker/setup-melo.sh | 77 ++++++++++++++++ docker/supervisord.conf | 20 +++++ 6 files changed, 361 insertions(+), 4 deletions(-) create mode 100644 bridge/melo_worker.py create mode 100755 docker/setup-melo.sh diff --git a/.env.example b/.env.example index fe8d1c1..8ac1a11 100644 --- a/.env.example +++ b/.env.example @@ -28,6 +28,18 @@ WHISPER_DEVICE=cuda WHISPER_COMPUTE_TYPE=float16 # Optional explicit Piper voice model (.onnx). If empty, the jarvis default is used. TTS_PIPER_MODEL_PATH= +# TTS engine: "melo" (default) uses the MeloTTS Korean voice served by the warm +# melo-worker (Korean speaker, speed 1.5). Set to "piper" to use Piper directly. +TTS_ENGINE=melo +# Melo-only by default: if MeloTTS synthesis fails the bridge returns no audio +# rather than speaking Korean through the English Piper voice (which mangles it). +# Set to 1 only if you explicitly want the Piper fallback. +MELO_FALLBACK_PIPER=0 +# Where the bridge reaches the in-container MeloTTS worker, and how long it +# waits for a synthesis. Speaking rate is set on the worker via MELO_SPEED. +MELO_WORKER_URL=http://127.0.0.1:8770 +MELO_TIMEOUT=30 +MELO_SPEED=1.5 # --------------------------------------------------------------------------- # Jarvis brain (Ollama-backed). In Docker these populate the rendered diff --git a/Dockerfile b/Dockerfile index 59001ca..dc6a873 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,6 +59,12 @@ RUN ls -d /opt/venv/lib/python*/site-packages/nvidia/cublas/lib \ > /etc/ld.so.conf.d/nvidia-cu12.conf 2>/dev/null \ && /sbin/ldconfig || true +# --- MeloTTS Korean voice (separate /opt/melo py3.11 venv; see setup-melo.sh). +# Heavy layer (torch CPU + transformers + MeCab); placed before the app +# COPY so it stays cached across source-only changes. --- +COPY docker/setup-melo.sh /app/docker/setup-melo.sh +RUN bash /app/docker/setup-melo.sh + # --- Discord bot deps (cache layer on lockfile) --- COPY bot/package.json bot/bun.lock /app/bot/ RUN cd /app/bot && bun install --frozen-lockfile || bun install diff --git a/bridge/melo_worker.py b/bridge/melo_worker.py new file mode 100644 index 0000000..073183e --- /dev/null +++ b/bridge/melo_worker.py @@ -0,0 +1,191 @@ +""" +MeloTTS worker +============== + +A tiny, dependency-light HTTP service that keeps a MeloTTS voice warm and +synthesises speech on demand. It runs in its OWN Python venv (``/opt/melo`` in +the container) so the heavy MeloTTS/torch/transformers stack stays isolated +from the slim brain-bridge venv (which pins ``numpy<2`` for faster-whisper). + +The bridge's ``synthesize()`` POSTs ``{"text": "..."}`` here and gets back a +16-bit PCM WAV. The MeloTTS model is loaded once at startup and reused, so each +request only pays inference cost, not model-load cost. + +Config (env): + MELO_WORKER_HOST bind host (default 127.0.0.1) + MELO_WORKER_PORT bind port (default 8770) + MELO_LANGUAGE MeloTTS language (default KR) + MELO_SPEED speaking rate (default 1.5 -> the approved "150") + MELO_DEVICE torch device (default cpu) + +Run: + /opt/melo/bin/python -m bridge.melo_worker +""" + +from __future__ import annotations + +import io +import json +import os +import sys +import tempfile +import threading +import wave +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +HOST = os.environ.get("MELO_WORKER_HOST", "127.0.0.1") +PORT = int(os.environ.get("MELO_WORKER_PORT", "8770")) +LANGUAGE = os.environ.get("MELO_LANGUAGE", "KR") +SPEED = float(os.environ.get("MELO_SPEED", "1.5")) +DEVICE = os.environ.get("MELO_DEVICE", "cpu") + +# Model + speaker id are loaded once, guarded by a lock because MeloTTS +# inference is not guaranteed thread-safe. +_model = None +_speaker_id = None +_model_lock = threading.Lock() +_load_error: str | None = None + + +def _ensure_model() -> None: + global _model, _speaker_id, _load_error + if _model is not None or _load_error is not None: + return + with _model_lock: + if _model is not None or _load_error is not None: + return + try: + from melo.api import TTS # type: ignore + + model = TTS(language=LANGUAGE, device=DEVICE) + # spk2id is a melo HParams object (dict-like, supports __getitem__, + # __contains__, keys) but NOT .get(). The KR model exposes a single + # 'KR' speaker; fall back to the first id for other languages. + spk_map = model.hps.data.spk2id + keys = list(spk_map.keys()) + speaker_id = spk_map[LANGUAGE] if LANGUAGE in spk_map else spk_map[keys[0]] + _model = model + _speaker_id = speaker_id + print( + f"[melo-worker] ready (lang={LANGUAGE} speed={SPEED} " + f"device={DEVICE} speakers={list(spk_map.keys())})", + flush=True, + ) + except Exception as e: # pragma: no cover - depends on local model files + _load_error = f"{type(e).__name__}: {e}" + print(f"[melo-worker] model load FAILED: {_load_error}", flush=True) + + +def _synthesize(text: str) -> bytes: + """Synthesise ``text`` to a 16-bit PCM WAV (bytes).""" + _ensure_model() + if _model is None: + raise RuntimeError(_load_error or "melo model unavailable") + # MeloTTS writes to a file via soundfile; render to a container-disk temp + # file (NOT tmpfs), read it back, then drop it. + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp: + tmp_path = tmp.name + try: + with _model_lock: + _model.tts_to_file(text, _speaker_id, tmp_path, speed=SPEED) + with open(tmp_path, "rb") as f: + raw = f.read() + finally: + try: + os.unlink(tmp_path) + except OSError: + pass + return _ensure_pcm16_wav(raw) + + +def _ensure_pcm16_wav(raw: bytes) -> bytes: + """Guarantee a 16-bit PCM WAV. MeloTTS/soundfile usually emit float WAVs; + the Discord playback path (ffmpeg) tolerates both, but we normalise to + PCM16 so the contract matches the previous Piper output.""" + try: + with wave.open(io.BytesIO(raw), "rb") as wf: + if wf.getsampwidth() == 2: + return raw # already PCM16 + except wave.Error: + pass + # Non-PCM16 (e.g. float) — convert with soundfile if available. + try: + import numpy as np + import soundfile as sf + + data, sr = sf.read(io.BytesIO(raw), dtype="float32") + if data.ndim > 1: + data = data.mean(axis=1) # mono + pcm = np.clip(data, -1.0, 1.0) + pcm = (pcm * 32767.0).astype(" None: + body = json.dumps(payload).encode("utf-8") + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): # noqa: N802 + if self.path == "/health": + _ensure_model() + ok = _model is not None + self._json(200 if ok else 503, {"ok": ok, "error": _load_error}) + else: + self._json(404, {"error": "not found"}) + + def do_POST(self): # noqa: N802 + if self.path != "/synth": + self._json(404, {"error": "not found"}) + return + try: + length = int(self.headers.get("Content-Length", "0")) + data = json.loads(self.rfile.read(length) or b"{}") + text = (data.get("text") or "").strip() + except Exception as e: + self._json(400, {"error": f"bad request: {e}"}) + return + if not text: + self._json(400, {"error": "missing 'text'"}) + return + try: + wav = _synthesize(text) + except Exception as e: + self._json(503, {"error": f"{type(e).__name__}: {e}"}) + return + self.send_response(200) + self.send_header("Content-Type", "audio/wav") + self.send_header("Content-Length", str(len(wav))) + self.end_headers() + self.wfile.write(wav) + + def log_message(self, *args): # silence default request logging + return + + +def main() -> int: + # Warm the model at startup so the first Discord turn isn't slow. + _ensure_model() + server = ThreadingHTTPServer((HOST, PORT), _Handler) + print(f"[melo-worker] listening on http://{HOST}:{PORT}", flush=True) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bridge/server.py b/bridge/server.py index f017788..367ab0b 100644 --- a/bridge/server.py +++ b/bridge/server.py @@ -29,6 +29,7 @@ from __future__ import annotations import base64 import io +import json import os import sys import threading @@ -54,6 +55,18 @@ BRIDGE_PORT = int(os.environ.get("BRIDGE_PORT", "8765")) BRAIN_ENABLED = os.environ.get("JARVIS_BRAIN_ENABLED", "1") not in ("0", "false", "False") TTS_ENABLED = os.environ.get("JARVIS_TTS_ENABLED", "1") not in ("0", "false", "False") +# TTS engine: "melo" (MeloTTS Korean speaker, the warm worker) is the primary +# voice; Piper is kept as a fallback if the worker is unreachable. Set +# TTS_ENGINE=piper to disable MeloTTS entirely. +TTS_ENGINE = os.environ.get("TTS_ENGINE", "melo").strip().lower() +MELO_WORKER_URL = os.environ.get("MELO_WORKER_URL", "http://127.0.0.1:8770") +MELO_TIMEOUT = float(os.environ.get("MELO_TIMEOUT", "30")) +# When MeloTTS is the engine, do NOT silently fall back to the English Piper +# voice on failure: speaking Korean text through an English voice produces +# mangled audio. Default is melo-only (return no audio on failure); set +# MELO_FALLBACK_PIPER=1 to opt into the Piper fallback. +MELO_FALLBACK_PIPER = os.environ.get("MELO_FALLBACK_PIPER", "0") in ("1", "true", "True", "yes", "on") + # --------------------------------------------------------------------------- # Lazy singletons. The first request pays the model-load cost; afterwards the # brain stays warm. A lock guards initialization so concurrent Discord events @@ -207,10 +220,29 @@ def _coerce_bool(value) -> Optional[bool]: return str(value).strip().lower() in ("1", "true", "yes", "on") -def synthesize(text: str) -> Optional[bytes]: - """Synthesize text to a 16-bit PCM WAV using Piper. Returns None if TTS off.""" - if not TTS_ENABLED or not text.strip(): - return None +def _melo_synthesize(text: str) -> Optional[bytes]: + """Synthesise via the warm MeloTTS worker (separate /opt/melo venv, Korean + speaker @ speed 1.5). Returns a 16-bit PCM WAV, or None on any failure so + the caller can fall back to Piper.""" + import urllib.request + + try: + req = urllib.request.Request( + f"{MELO_WORKER_URL}/synth", + data=json.dumps({"text": text}).encode("utf-8"), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=MELO_TIMEOUT) as resp: + if resp.status == 200: + return resp.read() + print(f"[bridge] melo worker HTTP {resp.status}", flush=True) + except Exception as e: # pragma: no cover - worker may be down + print(f"[bridge] melo worker unreachable: {e}", flush=True) + return None + + +def _piper_synthesize(text: str) -> Optional[bytes]: + """Fallback: synthesise with Piper (English voice). Returns WAV bytes.""" _ensure_piper() if _piper_voice is None: return None @@ -223,6 +255,24 @@ def synthesize(text: str) -> Optional[bytes]: return buf.getvalue() +def synthesize(text: str) -> Optional[bytes]: + """Synthesize text to a 16-bit PCM WAV. The primary voice is MeloTTS + (Korean speaker, speed 1.5) served by the warm melo worker; Piper is a + fallback if the worker is unavailable. Returns None if TTS is off.""" + if not TTS_ENABLED or not text.strip(): + return None + if TTS_ENGINE == "melo": + audio = _melo_synthesize(text) + if audio: + return audio + if not MELO_FALLBACK_PIPER: + # Melo-only: better silent than mangled English for Korean text. + print("[bridge] melo synth failed; no audio (Piper fallback disabled)", flush=True) + return None + print("[bridge] melo synth failed; falling back to Piper", flush=True) + return _piper_synthesize(text) + + # --------------------------------------------------------------------------- # HTTP endpoints # --------------------------------------------------------------------------- @@ -235,6 +285,7 @@ def health(): "brain_ready": _cfg is not None, "brain_error": _brain_error, "tts_enabled": TTS_ENABLED, + "tts_engine": TTS_ENGINE, } ) diff --git a/docker/setup-melo.sh b/docker/setup-melo.sh new file mode 100755 index 0000000..1a363d0 --- /dev/null +++ b/docker/setup-melo.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# ============================================================================ +# Install a dedicated MeloTTS (Korean voice) venv at /opt/melo. +# +# Why a SEPARATE venv (not the brain-bridge /opt/venv): +# - MeloTTS pins old deps (transformers 4.27.4 / tokenizers 0.13.3 / fugashi) +# whose binary wheels exist only for cp311, so we use python3.11 here even +# though the image's default interpreter is 3.12. +# - It isolates the heavy torch/transformers stack from the slim bridge env, +# which pins numpy<2 for faster-whisper. +# +# torch is pinned to the CPU build: TTS runs on CPU so the GPU stays reserved +# for Ollama + Whisper, and we avoid pulling multi-GB CUDA wheels. +# ============================================================================ +set -euxo pipefail + +export DEBIAN_FRONTEND=noninteractive + +apt-get update +# Build deps for fugashi / mecab-python3 + a system MeCab dict, plus python3.11. +apt-get install -y --no-install-recommends \ + software-properties-common build-essential pkg-config swig \ + libmecab-dev mecab mecab-ipadic-utf8 +add-apt-repository -y ppa:deadsnakes/ppa +apt-get update +apt-get install -y --no-install-recommends python3.11 python3.11-venv python3.11-dev +rm -rf /var/lib/apt/lists/* + +python3.11 -m venv /opt/melo +/opt/melo/bin/pip install --no-cache-dir --upgrade pip wheel setuptools + +# CPU-only torch first, so MeloTTS's unpinned `torch` dep is already satisfied +# and pip does not pull the CUDA build. Pinned for reproducible rebuilds (these +# are the versions the CPU index resolved when this layer was verified). +/opt/melo/bin/pip install --no-cache-dir torch==2.12.0 torchaudio==2.11.0 \ + --index-url https://download.pytorch.org/whl/cpu + +# MeloTTS from GitHub. The PyPI sdist is broken (its setup.py reads a +# requirements.txt that is not shipped in the sdist), so install from the repo. +# Pinned to a commit (not refs/heads/main) so rebuilds are reproducible. +/opt/melo/bin/pip install --no-cache-dir \ + "https://github.com/myshell-ai/MeloTTS/archive/209145371cff8fc3bd60d7be902ea69cbdb7965a.tar.gz" + +# Korean g2p backend. MeloTTS otherwise tries to pip-install this on the first +# Korean request, which fails in a network-isolated container at runtime. +/opt/melo/bin/pip install --no-cache-dir python-mecab-ko python-mecab-ko-dic + +# Remove the full `unidic` package (its dictionary is never downloaded, only a +# stub) so mecab-python3 falls back to the bundled `unidic_lite` dict. Without +# this, importing melo's Japanese module fails with a missing-mecabrc error. +/opt/melo/bin/pip uninstall -y unidic || true + +# Pre-cache every model asset MeloTTS pulls at runtime, so the worker starts +# offline and the first Discord turn pays no download cost. Importing melo.api +# fetches the Japanese (tohoku-nlp/bert-base-japanese-v3) and Korean +# (kykim/bert-kor-base) BERT tokenizers plus nltk g2p data; loading the KR voice +# downloads the OpenVoice KR config+checkpoint, and a real synth pulls the +# Korean BERT weights. All of these go through huggingface_hub. +# +# CRITICAL: at runtime docker-compose mounts the `whisper_cache` named volume +# over /root/.cache/huggingface (for faster-whisper). That volume would SHADOW +# anything baked into the default HF cache, so we pin the melo worker to a +# DEDICATED, non-volume cache dir (/opt/melo-cache) here AND in supervisord, and +# warm it once. nltk_data (/root/nltk_data) is not volume-mounted so it stays. +export HF_HOME=/opt/melo-cache +mkdir -p "$HF_HOME" +MELO_LANGUAGE=KR HF_HOME=/opt/melo-cache /opt/melo/bin/python - <<'PY' +import tempfile +from melo.api import TTS + +model = TTS(language="KR", device="cpu") +out = tempfile.mktemp(suffix=".wav") +model.tts_to_file("초기화 워밍업입니다.", model.hps.data.spk2id["KR"], out, speed=1.5) +print("[setup-melo] warm-up KR synth OK ->", out) +PY + +echo "[setup-melo] MeloTTS venv ready at /opt/melo" diff --git a/docker/supervisord.conf b/docker/supervisord.conf index d178a35..0e7d814 100644 --- a/docker/supervisord.conf +++ b/docker/supervisord.conf @@ -49,6 +49,26 @@ stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 +[program:melo-worker] +; Warm MeloTTS Korean voice (speed 1.5) in its own py3.11 venv. The bridge's +; synthesize() POSTs here; if this is down the bridge falls back to Piper. +command=/opt/melo/bin/python /app/bridge/melo_worker.py +directory=/app +; HF_HOME points at the dedicated, image-baked melo cache (warmed in +; setup-melo.sh). The brain's whisper_cache volume is mounted over +; /root/.cache/huggingface, so without this the pre-cached BERT + KR checkpoint +; would be shadowed and re-downloaded (and would fail if the host is offline). +; HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE force pure-cache reads: the pinned old +; transformers/huggingface_hub otherwise retry the network on every load and +; error out instead of falling back to the (complete) baked cache. +environment=MELO_LANGUAGE="KR",MELO_SPEED="1.5",MELO_DEVICE="cpu",MELO_WORKER_HOST="127.0.0.1",MELO_WORKER_PORT="8770",HF_HOME="/opt/melo-cache",HF_HUB_OFFLINE="1",TRANSFORMERS_OFFLINE="1" +priority=280 +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + [program:bridge] command=/opt/venv/bin/python -m bridge.server directory=/app