feat(tts): add MeloTTS Korean voice via warm worker with offline-baked cache
Adds a dedicated MeloTTS Korean voice (speed 1.5) as the primary TTS engine, served by a long-lived in-container worker so each Discord turn pays only inference cost, not model-load cost. - bridge/melo_worker.py: tiny HTTP service in its own /opt/melo py3.11 venv, keeps the KR model warm, returns PCM16 WAV on POST /synth. - bridge/server.py: synthesize() routes to the melo worker first; Piper stays as an opt-in fallback (MELO_FALLBACK_PIPER, default off so Korean is never mangled through the English voice). /health reports tts_engine. - docker/setup-melo.sh: builds the isolated venv (pinned torch 2.12.0 / torchaudio 2.11.0 CPU, MeloTTS pinned to a commit for reproducible rebuilds), pre-fetches mecab-ko, and warms a dedicated HF cache (/opt/melo-cache) with a real KR synth so all BERT + KR checkpoint assets are baked into the image. - docker/supervisord.conf: runs melo-worker before the bridge with HF_HOME=/opt/melo-cache (the whisper_cache volume shadows the default HF cache) plus HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE so it reads the baked cache and never retries the network on load. - Dockerfile/.env.example: wire the melo build layer and config knobs. Verified: offline synth passes with --network none and the prod volume mounted; prod container recreated, all supervisord services up, bot logged in, and an end-to-end /tts call returns a 44.1kHz mono PCM16 WAV. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
javis-bot
191
bridge/melo_worker.py
Normal file
191
bridge/melo_worker.py
Normal file
@@ -0,0 +1,191 @@
|
||||
"""
|
||||
MeloTTS worker
|
||||
==============
|
||||
|
||||
A tiny, dependency-light HTTP service that keeps a MeloTTS voice warm and
|
||||
synthesises speech on demand. It runs in its OWN Python venv (``/opt/melo`` in
|
||||
the container) so the heavy MeloTTS/torch/transformers stack stays isolated
|
||||
from the slim brain-bridge venv (which pins ``numpy<2`` for faster-whisper).
|
||||
|
||||
The bridge's ``synthesize()`` POSTs ``{"text": "..."}`` here and gets back a
|
||||
16-bit PCM WAV. The MeloTTS model is loaded once at startup and reused, so each
|
||||
request only pays inference cost, not model-load cost.
|
||||
|
||||
Config (env):
|
||||
MELO_WORKER_HOST bind host (default 127.0.0.1)
|
||||
MELO_WORKER_PORT bind port (default 8770)
|
||||
MELO_LANGUAGE MeloTTS language (default KR)
|
||||
MELO_SPEED speaking rate (default 1.5 -> the approved "150")
|
||||
MELO_DEVICE torch device (default cpu)
|
||||
|
||||
Run:
|
||||
/opt/melo/bin/python -m bridge.melo_worker
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import wave
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
HOST = os.environ.get("MELO_WORKER_HOST", "127.0.0.1")
|
||||
PORT = int(os.environ.get("MELO_WORKER_PORT", "8770"))
|
||||
LANGUAGE = os.environ.get("MELO_LANGUAGE", "KR")
|
||||
SPEED = float(os.environ.get("MELO_SPEED", "1.5"))
|
||||
DEVICE = os.environ.get("MELO_DEVICE", "cpu")
|
||||
|
||||
# Model + speaker id are loaded once, guarded by a lock because MeloTTS
|
||||
# inference is not guaranteed thread-safe.
|
||||
_model = None
|
||||
_speaker_id = None
|
||||
_model_lock = threading.Lock()
|
||||
_load_error: str | None = None
|
||||
|
||||
|
||||
def _ensure_model() -> None:
|
||||
global _model, _speaker_id, _load_error
|
||||
if _model is not None or _load_error is not None:
|
||||
return
|
||||
with _model_lock:
|
||||
if _model is not None or _load_error is not None:
|
||||
return
|
||||
try:
|
||||
from melo.api import TTS # type: ignore
|
||||
|
||||
model = TTS(language=LANGUAGE, device=DEVICE)
|
||||
# spk2id is a melo HParams object (dict-like, supports __getitem__,
|
||||
# __contains__, keys) but NOT .get(). The KR model exposes a single
|
||||
# 'KR' speaker; fall back to the first id for other languages.
|
||||
spk_map = model.hps.data.spk2id
|
||||
keys = list(spk_map.keys())
|
||||
speaker_id = spk_map[LANGUAGE] if LANGUAGE in spk_map else spk_map[keys[0]]
|
||||
_model = model
|
||||
_speaker_id = speaker_id
|
||||
print(
|
||||
f"[melo-worker] ready (lang={LANGUAGE} speed={SPEED} "
|
||||
f"device={DEVICE} speakers={list(spk_map.keys())})",
|
||||
flush=True,
|
||||
)
|
||||
except Exception as e: # pragma: no cover - depends on local model files
|
||||
_load_error = f"{type(e).__name__}: {e}"
|
||||
print(f"[melo-worker] model load FAILED: {_load_error}", flush=True)
|
||||
|
||||
|
||||
def _synthesize(text: str) -> bytes:
|
||||
"""Synthesise ``text`` to a 16-bit PCM WAV (bytes)."""
|
||||
_ensure_model()
|
||||
if _model is None:
|
||||
raise RuntimeError(_load_error or "melo model unavailable")
|
||||
# MeloTTS writes to a file via soundfile; render to a container-disk temp
|
||||
# file (NOT tmpfs), read it back, then drop it.
|
||||
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
with _model_lock:
|
||||
_model.tts_to_file(text, _speaker_id, tmp_path, speed=SPEED)
|
||||
with open(tmp_path, "rb") as f:
|
||||
raw = f.read()
|
||||
finally:
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
return _ensure_pcm16_wav(raw)
|
||||
|
||||
|
||||
def _ensure_pcm16_wav(raw: bytes) -> bytes:
|
||||
"""Guarantee a 16-bit PCM WAV. MeloTTS/soundfile usually emit float WAVs;
|
||||
the Discord playback path (ffmpeg) tolerates both, but we normalise to
|
||||
PCM16 so the contract matches the previous Piper output."""
|
||||
try:
|
||||
with wave.open(io.BytesIO(raw), "rb") as wf:
|
||||
if wf.getsampwidth() == 2:
|
||||
return raw # already PCM16
|
||||
except wave.Error:
|
||||
pass
|
||||
# Non-PCM16 (e.g. float) — convert with soundfile if available.
|
||||
try:
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
|
||||
data, sr = sf.read(io.BytesIO(raw), dtype="float32")
|
||||
if data.ndim > 1:
|
||||
data = data.mean(axis=1) # mono
|
||||
pcm = np.clip(data, -1.0, 1.0)
|
||||
pcm = (pcm * 32767.0).astype("<i2").tobytes()
|
||||
buf = io.BytesIO()
|
||||
with wave.open(buf, "wb") as wf:
|
||||
wf.setnchannels(1)
|
||||
wf.setsampwidth(2)
|
||||
wf.setframerate(int(sr))
|
||||
wf.writeframes(pcm)
|
||||
return buf.getvalue()
|
||||
except Exception:
|
||||
return raw # last resort: hand back whatever MeloTTS produced
|
||||
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
def _json(self, code: int, payload: dict) -> None:
|
||||
body = json.dumps(payload).encode("utf-8")
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_GET(self): # noqa: N802
|
||||
if self.path == "/health":
|
||||
_ensure_model()
|
||||
ok = _model is not None
|
||||
self._json(200 if ok else 503, {"ok": ok, "error": _load_error})
|
||||
else:
|
||||
self._json(404, {"error": "not found"})
|
||||
|
||||
def do_POST(self): # noqa: N802
|
||||
if self.path != "/synth":
|
||||
self._json(404, {"error": "not found"})
|
||||
return
|
||||
try:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
data = json.loads(self.rfile.read(length) or b"{}")
|
||||
text = (data.get("text") or "").strip()
|
||||
except Exception as e:
|
||||
self._json(400, {"error": f"bad request: {e}"})
|
||||
return
|
||||
if not text:
|
||||
self._json(400, {"error": "missing 'text'"})
|
||||
return
|
||||
try:
|
||||
wav = _synthesize(text)
|
||||
except Exception as e:
|
||||
self._json(503, {"error": f"{type(e).__name__}: {e}"})
|
||||
return
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "audio/wav")
|
||||
self.send_header("Content-Length", str(len(wav)))
|
||||
self.end_headers()
|
||||
self.wfile.write(wav)
|
||||
|
||||
def log_message(self, *args): # silence default request logging
|
||||
return
|
||||
|
||||
|
||||
def main() -> int:
|
||||
# Warm the model at startup so the first Discord turn isn't slow.
|
||||
_ensure_model()
|
||||
server = ThreadingHTTPServer((HOST, PORT), _Handler)
|
||||
print(f"[melo-worker] listening on http://{HOST}:{PORT}", flush=True)
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -29,6 +29,7 @@ from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
@@ -54,6 +55,18 @@ BRIDGE_PORT = int(os.environ.get("BRIDGE_PORT", "8765"))
|
||||
BRAIN_ENABLED = os.environ.get("JARVIS_BRAIN_ENABLED", "1") not in ("0", "false", "False")
|
||||
TTS_ENABLED = os.environ.get("JARVIS_TTS_ENABLED", "1") not in ("0", "false", "False")
|
||||
|
||||
# TTS engine: "melo" (MeloTTS Korean speaker, the warm worker) is the primary
|
||||
# voice; Piper is kept as a fallback if the worker is unreachable. Set
|
||||
# TTS_ENGINE=piper to disable MeloTTS entirely.
|
||||
TTS_ENGINE = os.environ.get("TTS_ENGINE", "melo").strip().lower()
|
||||
MELO_WORKER_URL = os.environ.get("MELO_WORKER_URL", "http://127.0.0.1:8770")
|
||||
MELO_TIMEOUT = float(os.environ.get("MELO_TIMEOUT", "30"))
|
||||
# When MeloTTS is the engine, do NOT silently fall back to the English Piper
|
||||
# voice on failure: speaking Korean text through an English voice produces
|
||||
# mangled audio. Default is melo-only (return no audio on failure); set
|
||||
# MELO_FALLBACK_PIPER=1 to opt into the Piper fallback.
|
||||
MELO_FALLBACK_PIPER = os.environ.get("MELO_FALLBACK_PIPER", "0") in ("1", "true", "True", "yes", "on")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lazy singletons. The first request pays the model-load cost; afterwards the
|
||||
# brain stays warm. A lock guards initialization so concurrent Discord events
|
||||
@@ -207,10 +220,29 @@ def _coerce_bool(value) -> Optional[bool]:
|
||||
return str(value).strip().lower() in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
def synthesize(text: str) -> Optional[bytes]:
|
||||
"""Synthesize text to a 16-bit PCM WAV using Piper. Returns None if TTS off."""
|
||||
if not TTS_ENABLED or not text.strip():
|
||||
return None
|
||||
def _melo_synthesize(text: str) -> Optional[bytes]:
|
||||
"""Synthesise via the warm MeloTTS worker (separate /opt/melo venv, Korean
|
||||
speaker @ speed 1.5). Returns a 16-bit PCM WAV, or None on any failure so
|
||||
the caller can fall back to Piper."""
|
||||
import urllib.request
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
f"{MELO_WORKER_URL}/synth",
|
||||
data=json.dumps({"text": text}).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=MELO_TIMEOUT) as resp:
|
||||
if resp.status == 200:
|
||||
return resp.read()
|
||||
print(f"[bridge] melo worker HTTP {resp.status}", flush=True)
|
||||
except Exception as e: # pragma: no cover - worker may be down
|
||||
print(f"[bridge] melo worker unreachable: {e}", flush=True)
|
||||
return None
|
||||
|
||||
|
||||
def _piper_synthesize(text: str) -> Optional[bytes]:
|
||||
"""Fallback: synthesise with Piper (English voice). Returns WAV bytes."""
|
||||
_ensure_piper()
|
||||
if _piper_voice is None:
|
||||
return None
|
||||
@@ -223,6 +255,24 @@ def synthesize(text: str) -> Optional[bytes]:
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def synthesize(text: str) -> Optional[bytes]:
|
||||
"""Synthesize text to a 16-bit PCM WAV. The primary voice is MeloTTS
|
||||
(Korean speaker, speed 1.5) served by the warm melo worker; Piper is a
|
||||
fallback if the worker is unavailable. Returns None if TTS is off."""
|
||||
if not TTS_ENABLED or not text.strip():
|
||||
return None
|
||||
if TTS_ENGINE == "melo":
|
||||
audio = _melo_synthesize(text)
|
||||
if audio:
|
||||
return audio
|
||||
if not MELO_FALLBACK_PIPER:
|
||||
# Melo-only: better silent than mangled English for Korean text.
|
||||
print("[bridge] melo synth failed; no audio (Piper fallback disabled)", flush=True)
|
||||
return None
|
||||
print("[bridge] melo synth failed; falling back to Piper", flush=True)
|
||||
return _piper_synthesize(text)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTTP endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -235,6 +285,7 @@ def health():
|
||||
"brain_ready": _cfg is not None,
|
||||
"brain_error": _brain_error,
|
||||
"tts_enabled": TTS_ENABLED,
|
||||
"tts_engine": TTS_ENGINE,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user