feat: replace MeloTTS with Coqui XTTS-v2 natural Korean voice
Some checks failed
Release / semantic-release (push) Successful in 30s
tests / Unit tests (Linux, Python 3.11) (push) Failing after 5m17s
Release / build-windows (push) Has been cancelled
Release / build-macos (arm64, macos-latest) (push) Has been cancelled
Release / build-macos (x64, macos-15-intel) (push) Has been cancelled
Release / release-main (push) Has been cancelled
Release / release-develop (push) Has been cancelled
Release / build-linux (push) Has been cancelled

MeloTTS's single Korean speaker sounded non-native ("foreign accent"). Swap it
for Coqui XTTS-v2 with the built-in female studio speaker "Ana Florence"
(language ko), the natural voice used in earlier local runs.

- bridge/xtts_worker.py: new warm HTTP worker (own /opt/xtts venv), same
  /synth + /health contract and PCM16 output as the old melo worker
- docker/setup-xtts.sh: builds the venv with cu128 torch (Blackwell) + Coqui
  TTS and bakes the XTTS-v2 model offline. Pins transformers>=4.57,<5 (5.x
  removed isin_mps_friendly, breaking XTTS) and installs the [codec] extra
  (torch>=2.9 needs torchcodec) — both verified by a real host synth
- Dockerfile: replace the melo build layer with the xtts layer
- supervisord.conf: melo-worker -> xtts-worker, env passthrough for
  XTTS_DEVICE/SPEAKER/LANGUAGE (always set via compose defaults)
- bridge/server.py: default TTS_ENGINE=xtts, route to the xtts worker, generic
  worker-synth helper, neural-only fallback flag (XTTS_FALLBACK_PIPER)
- settings UI: engine dropdown xtts/piper, drop the dead melo_speed field, fix
  the supervisorctl restart target to xtts-worker
- compose/.env.example/README: XTTS_* vars, speaker/language knobs, remove melo
- remove bridge/melo_worker.py and docker/setup-melo.sh
- tests: xtts treated as multilingual (not English-only)

Verified on host: coqui-tts loads XTTS-v2 and synthesises Korean as
"Ana Florence" to a 16-bit mono 24kHz WAV.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-06-23 03:08:01 +09:00
parent b9f637faa4
commit 39a0944105
11 changed files with 251 additions and 243 deletions

View File

@@ -87,12 +87,13 @@ VAD_MIN_SPEECH_MS = int(os.environ.get("VAD_MIN_SPEECH_MS", "200"))
# Korean phrase decoded as Chinese) and shaves a little latency. Empty = auto.
STT_LANGUAGE = os.environ.get("STT_LANGUAGE", "ko").strip() or None
# TTS engine: "melo" (MeloTTS Korean speaker, the warm worker) is the primary
# voice; Piper is kept as a fallback if the worker is unreachable. Set
# TTS_ENGINE=piper to disable MeloTTS entirely.
# TTS engine: "xtts" (Coqui XTTS-v2 natural Korean voice, the warm worker) is
# the primary voice; Piper is kept as a fallback only if explicitly enabled. Set
# TTS_ENGINE=piper to disable the neural Korean voice entirely. "melo" is still
# accepted for backward compatibility but is no longer built into the image.
def _tts_engine_setting() -> str:
"""TTS engine: settings-UI value (runtime config JSON) wins, else env, else
melo. Read at startup; the settings UI restarts the bridge on apply."""
xtts. Read at startup; the settings UI restarts the bridge on apply."""
try:
_cp = os.environ.get("JARVIS_CONFIG_PATH", "/app/config/jarvis.json")
_v = json.loads(open(_cp, encoding="utf-8").read()).get("tts_engine")
@@ -100,17 +101,29 @@ def _tts_engine_setting() -> str:
return str(_v).strip().lower()
except Exception:
pass
return os.environ.get("TTS_ENGINE", "melo").strip().lower()
return os.environ.get("TTS_ENGINE", "xtts").strip().lower()
TTS_ENGINE = _tts_engine_setting()
# Coqui XTTS-v2 worker (the natural Korean voice).
XTTS_WORKER_URL = os.environ.get("XTTS_WORKER_URL", "http://127.0.0.1:8771")
XTTS_TIMEOUT = float(os.environ.get("XTTS_TIMEOUT", "30"))
# Legacy MeloTTS worker (no longer built into the image; kept for back-compat
# if someone runs an old worker out-of-band).
MELO_WORKER_URL = os.environ.get("MELO_WORKER_URL", "http://127.0.0.1:8770")
MELO_TIMEOUT = float(os.environ.get("MELO_TIMEOUT", "30"))
# When MeloTTS is the engine, do NOT silently fall back to the English Piper
# voice on failure: speaking Korean text through an English voice produces
# mangled audio. Default is melo-only (return no audio on failure); set
# MELO_FALLBACK_PIPER=1 to opt into the Piper fallback.
MELO_FALLBACK_PIPER = os.environ.get("MELO_FALLBACK_PIPER", "0") in ("1", "true", "True", "yes", "on")
# Do NOT silently fall back to the English Piper voice on a neural-voice failure:
# speaking Korean text through an English voice produces mangled audio. Default
# is neural-only (return no audio on failure); set XTTS_FALLBACK_PIPER=1 (or the
# legacy MELO_FALLBACK_PIPER=1) to opt into the Piper fallback.
def _truthy_env(*names: str) -> bool:
for _n in names:
if os.environ.get(_n, "").strip().lower() in ("1", "true", "yes", "on"):
return True
return False
NEURAL_FALLBACK_PIPER = _truthy_env("XTTS_FALLBACK_PIPER", "MELO_FALLBACK_PIPER")
# ---------------------------------------------------------------------------
# Lazy singletons. The first request pays the model-load cost; afterwards the
@@ -302,27 +315,38 @@ def _coerce_bool(value) -> Optional[bool]:
return str(value).strip().lower() in ("1", "true", "yes", "on")
def _melo_synthesize(text: str) -> Optional[bytes]:
"""Synthesise via the warm MeloTTS worker (separate /opt/melo venv, Korean
speaker @ speed 1.5). Returns a 16-bit PCM WAV, or None on any failure so
the caller can fall back to Piper."""
def _worker_synthesize(name: str, url: str, timeout: float, text: str) -> Optional[bytes]:
"""POST text to a warm TTS worker's /synth and return its WAV bytes, or None
on any failure so the caller can decide whether to fall back."""
import urllib.request
try:
req = urllib.request.Request(
f"{MELO_WORKER_URL}/synth",
f"{url}/synth",
data=json.dumps({"text": text}).encode("utf-8"),
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(req, timeout=MELO_TIMEOUT) as resp:
with urllib.request.urlopen(req, timeout=timeout) as resp:
if resp.status == 200:
return resp.read()
print(f"[bridge] melo worker HTTP {resp.status}", flush=True)
print(f"[bridge] {name} worker HTTP {resp.status}", flush=True)
except Exception as e: # pragma: no cover - worker may be down
print(f"[bridge] melo worker unreachable: {e}", flush=True)
print(f"[bridge] {name} worker unreachable: {e}", flush=True)
return None
def _xtts_synthesize(text: str) -> Optional[bytes]:
"""Synthesise via the warm Coqui XTTS-v2 worker (separate /opt/xtts venv,
natural female Korean). Returns a 16-bit PCM WAV, or None on failure."""
return _worker_synthesize("xtts", XTTS_WORKER_URL, XTTS_TIMEOUT, text)
def _melo_synthesize(text: str) -> Optional[bytes]:
"""Legacy: synthesise via a MeloTTS worker if one is running out-of-band.
Returns a 16-bit PCM WAV, or None on any failure."""
return _worker_synthesize("melo", MELO_WORKER_URL, MELO_TIMEOUT, text)
def _piper_synthesize(text: str) -> Optional[bytes]:
"""Fallback: synthesise with Piper (English voice). Returns WAV bytes."""
_ensure_piper()
@@ -349,11 +373,12 @@ def _tts_ready() -> bool:
"""
if not TTS_ENABLED:
return True
if TTS_ENGINE == "melo":
_worker_health = {"xtts": XTTS_WORKER_URL, "melo": MELO_WORKER_URL}.get(TTS_ENGINE)
if _worker_health:
import urllib.request
try:
with urllib.request.urlopen(f"{MELO_WORKER_URL}/health", timeout=2) as resp:
with urllib.request.urlopen(f"{_worker_health}/health", timeout=2) as resp:
return resp.status == 200
except Exception:
return False
@@ -361,20 +386,24 @@ def _tts_ready() -> bool:
def synthesize(text: str) -> Optional[bytes]:
"""Synthesize text to a 16-bit PCM WAV. The primary voice is MeloTTS
(Korean speaker, speed 1.5) served by the warm melo worker; Piper is a
fallback if the worker is unavailable. Returns None if TTS is off."""
"""Synthesize text to a 16-bit PCM WAV. The primary voice is Coqui XTTS-v2
(natural female Korean) served by the warm xtts worker; Piper is used only
when explicitly enabled as a fallback. Returns None if TTS is off."""
if not TTS_ENABLED or not text.strip():
return None
if TTS_ENGINE == "melo":
audio = _melo_synthesize(text)
_neural = {"xtts": _xtts_synthesize, "melo": _melo_synthesize}.get(TTS_ENGINE)
if _neural is not None:
audio = _neural(text)
if audio:
return audio
if not MELO_FALLBACK_PIPER:
# Melo-only: better silent than mangled English for Korean text.
print("[bridge] melo synth failed; no audio (Piper fallback disabled)", flush=True)
if not NEURAL_FALLBACK_PIPER:
# Neural-only: better silent than mangled English for Korean text.
print(
f"[bridge] {TTS_ENGINE} synth failed; no audio (Piper fallback disabled)",
flush=True,
)
return None
print("[bridge] melo synth failed; falling back to Piper", flush=True)
print(f"[bridge] {TTS_ENGINE} synth failed; falling back to Piper", flush=True)
return _piper_synthesize(text)