feat: replace MeloTTS with Coqui XTTS-v2 natural Korean voice
Some checks failed
Release / semantic-release (push) Successful in 30s
tests / Unit tests (Linux, Python 3.11) (push) Failing after 5m17s
Release / build-windows (push) Has been cancelled
Release / build-macos (arm64, macos-latest) (push) Has been cancelled
Release / build-macos (x64, macos-15-intel) (push) Has been cancelled
Release / release-main (push) Has been cancelled
Release / release-develop (push) Has been cancelled
Release / build-linux (push) Has been cancelled
Some checks failed
Release / semantic-release (push) Successful in 30s
tests / Unit tests (Linux, Python 3.11) (push) Failing after 5m17s
Release / build-windows (push) Has been cancelled
Release / build-macos (arm64, macos-latest) (push) Has been cancelled
Release / build-macos (x64, macos-15-intel) (push) Has been cancelled
Release / release-main (push) Has been cancelled
Release / release-develop (push) Has been cancelled
Release / build-linux (push) Has been cancelled
MeloTTS's single Korean speaker sounded non-native ("foreign accent"). Swap it
for Coqui XTTS-v2 with the built-in female studio speaker "Ana Florence"
(language ko), the natural voice used in earlier local runs.
- bridge/xtts_worker.py: new warm HTTP worker (own /opt/xtts venv), same
/synth + /health contract and PCM16 output as the old melo worker
- docker/setup-xtts.sh: builds the venv with cu128 torch (Blackwell) + Coqui
TTS and bakes the XTTS-v2 model offline. Pins transformers>=4.57,<5 (5.x
removed isin_mps_friendly, breaking XTTS) and installs the [codec] extra
(torch>=2.9 needs torchcodec) — both verified by a real host synth
- Dockerfile: replace the melo build layer with the xtts layer
- supervisord.conf: melo-worker -> xtts-worker, env passthrough for
XTTS_DEVICE/SPEAKER/LANGUAGE (always set via compose defaults)
- bridge/server.py: default TTS_ENGINE=xtts, route to the xtts worker, generic
worker-synth helper, neural-only fallback flag (XTTS_FALLBACK_PIPER)
- settings UI: engine dropdown xtts/piper, drop the dead melo_speed field, fix
the supervisorctl restart target to xtts-worker
- compose/.env.example/README: XTTS_* vars, speaker/language knobs, remove melo
- remove bridge/melo_worker.py and docker/setup-melo.sh
- tests: xtts treated as multilingual (not English-only)
Verified on host: coqui-tts loads XTTS-v2 and synthesises Korean as
"Ana Florence" to a 16-bit mono 24kHz WAV.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -87,12 +87,13 @@ VAD_MIN_SPEECH_MS = int(os.environ.get("VAD_MIN_SPEECH_MS", "200"))
|
||||
# Korean phrase decoded as Chinese) and shaves a little latency. Empty = auto.
|
||||
STT_LANGUAGE = os.environ.get("STT_LANGUAGE", "ko").strip() or None
|
||||
|
||||
# TTS engine: "melo" (MeloTTS Korean speaker, the warm worker) is the primary
|
||||
# voice; Piper is kept as a fallback if the worker is unreachable. Set
|
||||
# TTS_ENGINE=piper to disable MeloTTS entirely.
|
||||
# TTS engine: "xtts" (Coqui XTTS-v2 natural Korean voice, the warm worker) is
|
||||
# the primary voice; Piper is kept as a fallback only if explicitly enabled. Set
|
||||
# TTS_ENGINE=piper to disable the neural Korean voice entirely. "melo" is still
|
||||
# accepted for backward compatibility but is no longer built into the image.
|
||||
def _tts_engine_setting() -> str:
|
||||
"""TTS engine: settings-UI value (runtime config JSON) wins, else env, else
|
||||
melo. Read at startup; the settings UI restarts the bridge on apply."""
|
||||
xtts. Read at startup; the settings UI restarts the bridge on apply."""
|
||||
try:
|
||||
_cp = os.environ.get("JARVIS_CONFIG_PATH", "/app/config/jarvis.json")
|
||||
_v = json.loads(open(_cp, encoding="utf-8").read()).get("tts_engine")
|
||||
@@ -100,17 +101,29 @@ def _tts_engine_setting() -> str:
|
||||
return str(_v).strip().lower()
|
||||
except Exception:
|
||||
pass
|
||||
return os.environ.get("TTS_ENGINE", "melo").strip().lower()
|
||||
return os.environ.get("TTS_ENGINE", "xtts").strip().lower()
|
||||
|
||||
|
||||
TTS_ENGINE = _tts_engine_setting()
|
||||
# Coqui XTTS-v2 worker (the natural Korean voice).
|
||||
XTTS_WORKER_URL = os.environ.get("XTTS_WORKER_URL", "http://127.0.0.1:8771")
|
||||
XTTS_TIMEOUT = float(os.environ.get("XTTS_TIMEOUT", "30"))
|
||||
# Legacy MeloTTS worker (no longer built into the image; kept for back-compat
|
||||
# if someone runs an old worker out-of-band).
|
||||
MELO_WORKER_URL = os.environ.get("MELO_WORKER_URL", "http://127.0.0.1:8770")
|
||||
MELO_TIMEOUT = float(os.environ.get("MELO_TIMEOUT", "30"))
|
||||
# When MeloTTS is the engine, do NOT silently fall back to the English Piper
|
||||
# voice on failure: speaking Korean text through an English voice produces
|
||||
# mangled audio. Default is melo-only (return no audio on failure); set
|
||||
# MELO_FALLBACK_PIPER=1 to opt into the Piper fallback.
|
||||
MELO_FALLBACK_PIPER = os.environ.get("MELO_FALLBACK_PIPER", "0") in ("1", "true", "True", "yes", "on")
|
||||
# Do NOT silently fall back to the English Piper voice on a neural-voice failure:
|
||||
# speaking Korean text through an English voice produces mangled audio. Default
|
||||
# is neural-only (return no audio on failure); set XTTS_FALLBACK_PIPER=1 (or the
|
||||
# legacy MELO_FALLBACK_PIPER=1) to opt into the Piper fallback.
|
||||
def _truthy_env(*names: str) -> bool:
|
||||
for _n in names:
|
||||
if os.environ.get(_n, "").strip().lower() in ("1", "true", "yes", "on"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
NEURAL_FALLBACK_PIPER = _truthy_env("XTTS_FALLBACK_PIPER", "MELO_FALLBACK_PIPER")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lazy singletons. The first request pays the model-load cost; afterwards the
|
||||
@@ -302,27 +315,38 @@ def _coerce_bool(value) -> Optional[bool]:
|
||||
return str(value).strip().lower() in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
def _melo_synthesize(text: str) -> Optional[bytes]:
|
||||
"""Synthesise via the warm MeloTTS worker (separate /opt/melo venv, Korean
|
||||
speaker @ speed 1.5). Returns a 16-bit PCM WAV, or None on any failure so
|
||||
the caller can fall back to Piper."""
|
||||
def _worker_synthesize(name: str, url: str, timeout: float, text: str) -> Optional[bytes]:
|
||||
"""POST text to a warm TTS worker's /synth and return its WAV bytes, or None
|
||||
on any failure so the caller can decide whether to fall back."""
|
||||
import urllib.request
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
f"{MELO_WORKER_URL}/synth",
|
||||
f"{url}/synth",
|
||||
data=json.dumps({"text": text}).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=MELO_TIMEOUT) as resp:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
if resp.status == 200:
|
||||
return resp.read()
|
||||
print(f"[bridge] melo worker HTTP {resp.status}", flush=True)
|
||||
print(f"[bridge] {name} worker HTTP {resp.status}", flush=True)
|
||||
except Exception as e: # pragma: no cover - worker may be down
|
||||
print(f"[bridge] melo worker unreachable: {e}", flush=True)
|
||||
print(f"[bridge] {name} worker unreachable: {e}", flush=True)
|
||||
return None
|
||||
|
||||
|
||||
def _xtts_synthesize(text: str) -> Optional[bytes]:
|
||||
"""Synthesise via the warm Coqui XTTS-v2 worker (separate /opt/xtts venv,
|
||||
natural female Korean). Returns a 16-bit PCM WAV, or None on failure."""
|
||||
return _worker_synthesize("xtts", XTTS_WORKER_URL, XTTS_TIMEOUT, text)
|
||||
|
||||
|
||||
def _melo_synthesize(text: str) -> Optional[bytes]:
|
||||
"""Legacy: synthesise via a MeloTTS worker if one is running out-of-band.
|
||||
Returns a 16-bit PCM WAV, or None on any failure."""
|
||||
return _worker_synthesize("melo", MELO_WORKER_URL, MELO_TIMEOUT, text)
|
||||
|
||||
|
||||
def _piper_synthesize(text: str) -> Optional[bytes]:
|
||||
"""Fallback: synthesise with Piper (English voice). Returns WAV bytes."""
|
||||
_ensure_piper()
|
||||
@@ -349,11 +373,12 @@ def _tts_ready() -> bool:
|
||||
"""
|
||||
if not TTS_ENABLED:
|
||||
return True
|
||||
if TTS_ENGINE == "melo":
|
||||
_worker_health = {"xtts": XTTS_WORKER_URL, "melo": MELO_WORKER_URL}.get(TTS_ENGINE)
|
||||
if _worker_health:
|
||||
import urllib.request
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(f"{MELO_WORKER_URL}/health", timeout=2) as resp:
|
||||
with urllib.request.urlopen(f"{_worker_health}/health", timeout=2) as resp:
|
||||
return resp.status == 200
|
||||
except Exception:
|
||||
return False
|
||||
@@ -361,20 +386,24 @@ def _tts_ready() -> bool:
|
||||
|
||||
|
||||
def synthesize(text: str) -> Optional[bytes]:
|
||||
"""Synthesize text to a 16-bit PCM WAV. The primary voice is MeloTTS
|
||||
(Korean speaker, speed 1.5) served by the warm melo worker; Piper is a
|
||||
fallback if the worker is unavailable. Returns None if TTS is off."""
|
||||
"""Synthesize text to a 16-bit PCM WAV. The primary voice is Coqui XTTS-v2
|
||||
(natural female Korean) served by the warm xtts worker; Piper is used only
|
||||
when explicitly enabled as a fallback. Returns None if TTS is off."""
|
||||
if not TTS_ENABLED or not text.strip():
|
||||
return None
|
||||
if TTS_ENGINE == "melo":
|
||||
audio = _melo_synthesize(text)
|
||||
_neural = {"xtts": _xtts_synthesize, "melo": _melo_synthesize}.get(TTS_ENGINE)
|
||||
if _neural is not None:
|
||||
audio = _neural(text)
|
||||
if audio:
|
||||
return audio
|
||||
if not MELO_FALLBACK_PIPER:
|
||||
# Melo-only: better silent than mangled English for Korean text.
|
||||
print("[bridge] melo synth failed; no audio (Piper fallback disabled)", flush=True)
|
||||
if not NEURAL_FALLBACK_PIPER:
|
||||
# Neural-only: better silent than mangled English for Korean text.
|
||||
print(
|
||||
f"[bridge] {TTS_ENGINE} synth failed; no audio (Piper fallback disabled)",
|
||||
flush=True,
|
||||
)
|
||||
return None
|
||||
print("[bridge] melo synth failed; falling back to Piper", flush=True)
|
||||
print(f"[bridge] {TTS_ENGINE} synth failed; falling back to Piper", flush=True)
|
||||
return _piper_synthesize(text)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user