feat(tts): add MeloTTS Korean voice via warm worker with offline-baked cache

Adds a dedicated MeloTTS Korean voice (speed 1.5) as the primary TTS engine, served by a long-lived in-container worker so each Discord turn pays only inference cost, not model-load cost. - bridge/melo_worker.py: tiny HTTP service in its own /opt/melo py3.11 venv, keeps the KR model warm, returns PCM16 WAV on POST /synth. - bridge/server.py: synthesize() routes to the melo worker first; Piper stays as an opt-in fallback (MELO_FALLBACK_PIPER, default off so Korean is never mangled through the English voice). /health reports tts_engine. - docker/setup-melo.sh: builds the isolated venv (pinned torch 2.12.0 / torchaudio 2.11.0 CPU, MeloTTS pinned to a commit for reproducible rebuilds), pre-fetches mecab-ko, and warms a dedicated HF cache (/opt/melo-cache) with a real KR synth so all BERT + KR checkpoint assets are baked into the image. - docker/supervisord.conf: runs melo-worker before the bridge with HF_HOME=/opt/melo-cache (the whisper_cache volume shadows the default HF cache) plus HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE so it reads the baked cache and never retries the network on load. - Dockerfile/.env.example: wire the melo build layer and config knobs. Verified: offline synth passes with --network none and the prod volume mounted; prod container recreated, all supervisord services up, bot logged in, and an end-to-end /tts call returns a 44.1kHz mono PCM16 WAV. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-06-12 19:01:54 +09:00
parent 3e333763fb
commit b17961e9e3
6 changed files with 361 additions and 4 deletions
--- a/.env.example
+++ b/.env.example
@@ -28,6 +28,18 @@ WHISPER_DEVICE=cuda
 WHISPER_COMPUTE_TYPE=float16
 # Optional explicit Piper voice model (.onnx). If empty, the jarvis default is used.
 TTS_PIPER_MODEL_PATH=
+# TTS engine: "melo" (default) uses the MeloTTS Korean voice served by the warm
+# melo-worker (Korean speaker, speed 1.5). Set to "piper" to use Piper directly.
+TTS_ENGINE=melo
+# Melo-only by default: if MeloTTS synthesis fails the bridge returns no audio
+# rather than speaking Korean through the English Piper voice (which mangles it).
+# Set to 1 only if you explicitly want the Piper fallback.
+MELO_FALLBACK_PIPER=0
+# Where the bridge reaches the in-container MeloTTS worker, and how long it
+# waits for a synthesis. Speaking rate is set on the worker via MELO_SPEED.
+MELO_WORKER_URL=http://127.0.0.1:8770
+MELO_TIMEOUT=30
+MELO_SPEED=1.5

 # ---------------------------------------------------------------------------
 # Jarvis brain (Ollama-backed). In Docker these populate the rendered
--- a/6
+++ b/6
@@ -59,6 +59,12 @@ RUN ls -d /opt/venv/lib/python*/site-packages/nvidia/cublas/lib \
        > /etc/ld.so.conf.d/nvidia-cu12.conf 2>/dev/null \
    && /sbin/ldconfig || true

+# --- MeloTTS Korean voice (separate /opt/melo py3.11 venv; see setup-melo.sh).
+#     Heavy layer (torch CPU + transformers + MeCab); placed before the app
+#     COPY so it stays cached across source-only changes. ---
+COPY docker/setup-melo.sh /app/docker/setup-melo.sh
+RUN bash /app/docker/setup-melo.sh
+
 # --- Discord bot deps (cache layer on lockfile) ---
 COPY bot/package.json bot/bun.lock /app/bot/
 RUN cd /app/bot && bun install --frozen-lockfile || bun install
--- a/bridge/melo_worker.py
+++ b/bridge/melo_worker.py
@@ -0,0 +1,191 @@
+"""
+MeloTTS worker
+==============
+
+A tiny, dependency-light HTTP service that keeps a MeloTTS voice warm and
+synthesises speech on demand. It runs in its OWN Python venv (``/opt/melo`` in
+the container) so the heavy MeloTTS/torch/transformers stack stays isolated
+from the slim brain-bridge venv (which pins ``numpy<2`` for faster-whisper).
+
+The bridge's ``synthesize()`` POSTs ``{"text": "..."}`` here and gets back a
+16-bit PCM WAV. The MeloTTS model is loaded once at startup and reused, so each
+request only pays inference cost, not model-load cost.
+
+Config (env):
+    MELO_WORKER_HOST   bind host          (default 127.0.0.1)
+    MELO_WORKER_PORT   bind port          (default 8770)
+    MELO_LANGUAGE      MeloTTS language   (default KR)
+    MELO_SPEED         speaking rate      (default 1.5  -> the approved "150")
+    MELO_DEVICE        torch device       (default cpu)
+
+Run:
+    /opt/melo/bin/python -m bridge.melo_worker
+"""
+
+from __future__ import annotations
+
+import io
+import json
+import os
+import sys
+import tempfile
+import threading
+import wave
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+HOST = os.environ.get("MELO_WORKER_HOST", "127.0.0.1")
+PORT = int(os.environ.get("MELO_WORKER_PORT", "8770"))
+LANGUAGE = os.environ.get("MELO_LANGUAGE", "KR")
+SPEED = float(os.environ.get("MELO_SPEED", "1.5"))
+DEVICE = os.environ.get("MELO_DEVICE", "cpu")
+
+# Model + speaker id are loaded once, guarded by a lock because MeloTTS
+# inference is not guaranteed thread-safe.
+_model = None
+_speaker_id = None
+_model_lock = threading.Lock()
+_load_error: str | None = None
+
+
+def _ensure_model() -> None:
+    global _model, _speaker_id, _load_error
+    if _model is not None or _load_error is not None:
+        return
+    with _model_lock:
+        if _model is not None or _load_error is not None:
+            return
+        try:
+            from melo.api import TTS  # type: ignore
+
+            model = TTS(language=LANGUAGE, device=DEVICE)
+            # spk2id is a melo HParams object (dict-like, supports __getitem__,
+            # __contains__, keys) but NOT .get(). The KR model exposes a single
+            # 'KR' speaker; fall back to the first id for other languages.
+            spk_map = model.hps.data.spk2id
+            keys = list(spk_map.keys())
+            speaker_id = spk_map[LANGUAGE] if LANGUAGE in spk_map else spk_map[keys[0]]
+            _model = model
+            _speaker_id = speaker_id
+            print(
+                f"[melo-worker] ready (lang={LANGUAGE} speed={SPEED} "
+                f"device={DEVICE} speakers={list(spk_map.keys())})",
+                flush=True,
+            )
+        except Exception as e:  # pragma: no cover - depends on local model files
+            _load_error = f"{type(e).__name__}: {e}"
+            print(f"[melo-worker] model load FAILED: {_load_error}", flush=True)
+
+
+def _synthesize(text: str) -> bytes:
+    """Synthesise ``text`` to a 16-bit PCM WAV (bytes)."""
+    _ensure_model()
+    if _model is None:
+        raise RuntimeError(_load_error or "melo model unavailable")
+    # MeloTTS writes to a file via soundfile; render to a container-disk temp
+    # file (NOT tmpfs), read it back, then drop it.
+    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
+        tmp_path = tmp.name
+    try:
+        with _model_lock:
+            _model.tts_to_file(text, _speaker_id, tmp_path, speed=SPEED)
+        with open(tmp_path, "rb") as f:
+            raw = f.read()
+    finally:
+        try:
+            os.unlink(tmp_path)
+        except OSError:
+            pass
+    return _ensure_pcm16_wav(raw)
+
+
+def _ensure_pcm16_wav(raw: bytes) -> bytes:
+    """Guarantee a 16-bit PCM WAV. MeloTTS/soundfile usually emit float WAVs;
+    the Discord playback path (ffmpeg) tolerates both, but we normalise to
+    PCM16 so the contract matches the previous Piper output."""
+    try:
+        with wave.open(io.BytesIO(raw), "rb") as wf:
+            if wf.getsampwidth() == 2:
+                return raw  # already PCM16
+    except wave.Error:
+        pass
+    # Non-PCM16 (e.g. float) — convert with soundfile if available.
+    try:
+        import numpy as np
+        import soundfile as sf
+
+        data, sr = sf.read(io.BytesIO(raw), dtype="float32")
+        if data.ndim > 1:
+            data = data.mean(axis=1)  # mono
+        pcm = np.clip(data, -1.0, 1.0)
+        pcm = (pcm * 32767.0).astype("<i2").tobytes()
+        buf = io.BytesIO()
+        with wave.open(buf, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)
+            wf.setframerate(int(sr))
+            wf.writeframes(pcm)
+        return buf.getvalue()
+    except Exception:
+        return raw  # last resort: hand back whatever MeloTTS produced
+
+
+class _Handler(BaseHTTPRequestHandler):
+    def _json(self, code: int, payload: dict) -> None:
+        body = json.dumps(payload).encode("utf-8")
+        self.send_response(code)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def do_GET(self):  # noqa: N802
+        if self.path == "/health":
+            _ensure_model()
+            ok = _model is not None
+            self._json(200 if ok else 503, {"ok": ok, "error": _load_error})
+        else:
+            self._json(404, {"error": "not found"})
+
+    def do_POST(self):  # noqa: N802
+        if self.path != "/synth":
+            self._json(404, {"error": "not found"})
+            return
+        try:
+            length = int(self.headers.get("Content-Length", "0"))
+            data = json.loads(self.rfile.read(length) or b"{}")
+            text = (data.get("text") or "").strip()
+        except Exception as e:
+            self._json(400, {"error": f"bad request: {e}"})
+            return
+        if not text:
+            self._json(400, {"error": "missing 'text'"})
+            return
+        try:
+            wav = _synthesize(text)
+        except Exception as e:
+            self._json(503, {"error": f"{type(e).__name__}: {e}"})
+            return
+        self.send_response(200)
+        self.send_header("Content-Type", "audio/wav")
+        self.send_header("Content-Length", str(len(wav)))
+        self.end_headers()
+        self.wfile.write(wav)
+
+    def log_message(self, *args):  # silence default request logging
+        return
+
+
+def main() -> int:
+    # Warm the model at startup so the first Discord turn isn't slow.
+    _ensure_model()
+    server = ThreadingHTTPServer((HOST, PORT), _Handler)
+    print(f"[melo-worker] listening on http://{HOST}:{PORT}", flush=True)
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
--- a/bridge/server.py
+++ b/bridge/server.py
@@ -29,6 +29,7 @@ from __future__ import annotations

 import base64
 import io
+import json
 import os
 import sys
 import threading
@@ -54,6 +55,18 @@ BRIDGE_PORT = int(os.environ.get("BRIDGE_PORT", "8765"))
 BRAIN_ENABLED = os.environ.get("JARVIS_BRAIN_ENABLED", "1") not in ("0", "false", "False")
 TTS_ENABLED = os.environ.get("JARVIS_TTS_ENABLED", "1") not in ("0", "false", "False")

+# TTS engine: "melo" (MeloTTS Korean speaker, the warm worker) is the primary
+# voice; Piper is kept as a fallback if the worker is unreachable. Set
+# TTS_ENGINE=piper to disable MeloTTS entirely.
+TTS_ENGINE = os.environ.get("TTS_ENGINE", "melo").strip().lower()
+MELO_WORKER_URL = os.environ.get("MELO_WORKER_URL", "http://127.0.0.1:8770")
+MELO_TIMEOUT = float(os.environ.get("MELO_TIMEOUT", "30"))
+# When MeloTTS is the engine, do NOT silently fall back to the English Piper
+# voice on failure: speaking Korean text through an English voice produces
+# mangled audio. Default is melo-only (return no audio on failure); set
+# MELO_FALLBACK_PIPER=1 to opt into the Piper fallback.
+MELO_FALLBACK_PIPER = os.environ.get("MELO_FALLBACK_PIPER", "0") in ("1", "true", "True", "yes", "on")
+
 # ---------------------------------------------------------------------------
 # Lazy singletons. The first request pays the model-load cost; afterwards the
 # brain stays warm. A lock guards initialization so concurrent Discord events
@@ -207,10 +220,29 @@ def _coerce_bool(value) -> Optional[bool]:
    return str(value).strip().lower() in ("1", "true", "yes", "on")


-def synthesize(text: str) -> Optional[bytes]:
-    """Synthesize text to a 16-bit PCM WAV using Piper. Returns None if TTS off."""
-    if not TTS_ENABLED or not text.strip():
-        return None
+def _melo_synthesize(text: str) -> Optional[bytes]:
+    """Synthesise via the warm MeloTTS worker (separate /opt/melo venv, Korean
+    speaker @ speed 1.5). Returns a 16-bit PCM WAV, or None on any failure so
+    the caller can fall back to Piper."""
+    import urllib.request
+
+    try:
+        req = urllib.request.Request(
+            f"{MELO_WORKER_URL}/synth",
+            data=json.dumps({"text": text}).encode("utf-8"),
+            headers={"Content-Type": "application/json"},
+        )
+        with urllib.request.urlopen(req, timeout=MELO_TIMEOUT) as resp:
+            if resp.status == 200:
+                return resp.read()
+            print(f"[bridge] melo worker HTTP {resp.status}", flush=True)
+    except Exception as e:  # pragma: no cover - worker may be down
+        print(f"[bridge] melo worker unreachable: {e}", flush=True)
+    return None
+
+
+def _piper_synthesize(text: str) -> Optional[bytes]:
+    """Fallback: synthesise with Piper (English voice). Returns WAV bytes."""
    _ensure_piper()
    if _piper_voice is None:
        return None
@@ -223,6 +255,24 @@ def synthesize(text: str) -> Optional[bytes]:
    return buf.getvalue()


+def synthesize(text: str) -> Optional[bytes]:
+    """Synthesize text to a 16-bit PCM WAV. The primary voice is MeloTTS
+    (Korean speaker, speed 1.5) served by the warm melo worker; Piper is a
+    fallback if the worker is unavailable. Returns None if TTS is off."""
+    if not TTS_ENABLED or not text.strip():
+        return None
+    if TTS_ENGINE == "melo":
+        audio = _melo_synthesize(text)
+        if audio:
+            return audio
+        if not MELO_FALLBACK_PIPER:
+            # Melo-only: better silent than mangled English for Korean text.
+            print("[bridge] melo synth failed; no audio (Piper fallback disabled)", flush=True)
+            return None
+        print("[bridge] melo synth failed; falling back to Piper", flush=True)
+    return _piper_synthesize(text)
+
+
 # ---------------------------------------------------------------------------
 # HTTP endpoints
 # ---------------------------------------------------------------------------
@@ -235,6 +285,7 @@ def health():
            "brain_ready": _cfg is not None,
            "brain_error": _brain_error,
            "tts_enabled": TTS_ENABLED,
+            "tts_engine": TTS_ENGINE,
        }
    )

--- a/docker/setup-melo.sh
+++ b/docker/setup-melo.sh
@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# ============================================================================
+# Install a dedicated MeloTTS (Korean voice) venv at /opt/melo.
+#
+# Why a SEPARATE venv (not the brain-bridge /opt/venv):
+#   - MeloTTS pins old deps (transformers 4.27.4 / tokenizers 0.13.3 / fugashi)
+#     whose binary wheels exist only for cp311, so we use python3.11 here even
+#     though the image's default interpreter is 3.12.
+#   - It isolates the heavy torch/transformers stack from the slim bridge env,
+#     which pins numpy<2 for faster-whisper.
+#
+# torch is pinned to the CPU build: TTS runs on CPU so the GPU stays reserved
+# for Ollama + Whisper, and we avoid pulling multi-GB CUDA wheels.
+# ============================================================================
+set -euxo pipefail
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update
+# Build deps for fugashi / mecab-python3 + a system MeCab dict, plus python3.11.
+apt-get install -y --no-install-recommends \
+    software-properties-common build-essential pkg-config swig \
+    libmecab-dev mecab mecab-ipadic-utf8
+add-apt-repository -y ppa:deadsnakes/ppa
+apt-get update
+apt-get install -y --no-install-recommends python3.11 python3.11-venv python3.11-dev
+rm -rf /var/lib/apt/lists/*
+
+python3.11 -m venv /opt/melo
+/opt/melo/bin/pip install --no-cache-dir --upgrade pip wheel setuptools
+
+# CPU-only torch first, so MeloTTS's unpinned `torch` dep is already satisfied
+# and pip does not pull the CUDA build. Pinned for reproducible rebuilds (these
+# are the versions the CPU index resolved when this layer was verified).
+/opt/melo/bin/pip install --no-cache-dir torch==2.12.0 torchaudio==2.11.0 \
+    --index-url https://download.pytorch.org/whl/cpu
+
+# MeloTTS from GitHub. The PyPI sdist is broken (its setup.py reads a
+# requirements.txt that is not shipped in the sdist), so install from the repo.
+# Pinned to a commit (not refs/heads/main) so rebuilds are reproducible.
+/opt/melo/bin/pip install --no-cache-dir \
+    "https://github.com/myshell-ai/MeloTTS/archive/209145371cff8fc3bd60d7be902ea69cbdb7965a.tar.gz"
+
+# Korean g2p backend. MeloTTS otherwise tries to pip-install this on the first
+# Korean request, which fails in a network-isolated container at runtime.
+/opt/melo/bin/pip install --no-cache-dir python-mecab-ko python-mecab-ko-dic
+
+# Remove the full `unidic` package (its dictionary is never downloaded, only a
+# stub) so mecab-python3 falls back to the bundled `unidic_lite` dict. Without
+# this, importing melo's Japanese module fails with a missing-mecabrc error.
+/opt/melo/bin/pip uninstall -y unidic || true
+
+# Pre-cache every model asset MeloTTS pulls at runtime, so the worker starts
+# offline and the first Discord turn pays no download cost. Importing melo.api
+# fetches the Japanese (tohoku-nlp/bert-base-japanese-v3) and Korean
+# (kykim/bert-kor-base) BERT tokenizers plus nltk g2p data; loading the KR voice
+# downloads the OpenVoice KR config+checkpoint, and a real synth pulls the
+# Korean BERT weights. All of these go through huggingface_hub.
+#
+# CRITICAL: at runtime docker-compose mounts the `whisper_cache` named volume
+# over /root/.cache/huggingface (for faster-whisper). That volume would SHADOW
+# anything baked into the default HF cache, so we pin the melo worker to a
+# DEDICATED, non-volume cache dir (/opt/melo-cache) here AND in supervisord, and
+# warm it once. nltk_data (/root/nltk_data) is not volume-mounted so it stays.
+export HF_HOME=/opt/melo-cache
+mkdir -p "$HF_HOME"
+MELO_LANGUAGE=KR HF_HOME=/opt/melo-cache /opt/melo/bin/python - <<'PY'
+import tempfile
+from melo.api import TTS
+
+model = TTS(language="KR", device="cpu")
+out = tempfile.mktemp(suffix=".wav")
+model.tts_to_file("초기화 워밍업입니다.", model.hps.data.spk2id["KR"], out, speed=1.5)
+print("[setup-melo] warm-up KR synth OK ->", out)
+PY
+
+echo "[setup-melo] MeloTTS venv ready at /opt/melo"
--- a/docker/supervisord.conf
+++ b/docker/supervisord.conf
@@ -49,6 +49,26 @@ stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0

+[program:melo-worker]
+; Warm MeloTTS Korean voice (speed 1.5) in its own py3.11 venv. The bridge's
+; synthesize() POSTs here; if this is down the bridge falls back to Piper.
+command=/opt/melo/bin/python /app/bridge/melo_worker.py
+directory=/app
+; HF_HOME points at the dedicated, image-baked melo cache (warmed in
+; setup-melo.sh). The brain's whisper_cache volume is mounted over
+; /root/.cache/huggingface, so without this the pre-cached BERT + KR checkpoint
+; would be shadowed and re-downloaded (and would fail if the host is offline).
+; HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE force pure-cache reads: the pinned old
+; transformers/huggingface_hub otherwise retry the network on every load and
+; error out instead of falling back to the (complete) baked cache.
+environment=MELO_LANGUAGE="KR",MELO_SPEED="1.5",MELO_DEVICE="cpu",MELO_WORKER_HOST="127.0.0.1",MELO_WORKER_PORT="8770",HF_HOME="/opt/melo-cache",HF_HUB_OFFLINE="1",TRANSFORMERS_OFFLINE="1"
+priority=280
+autorestart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
 [program:bridge]
 command=/opt/venv/bin/python -m bridge.server
 directory=/app