feat(tts): real Korean TTS via persistent MeloTTS worker
Adds a MeloTTS backend that runs the model in its own melo311 interpreter as a long-lived worker (melo_worker.py), loaded once and fed synthesis requests over a stdin/stdout JSON protocol. fd1 is split from fd2 in the worker so MeloTTS's stdout progress chatter can't corrupt the protocol. Each speak() writes a wav and hands the path to a pluggable sink (the Discord voice step will swap in "play into the call"). factory wires tts=melo; pipeline.aclose now also tears down the tts worker. Verified (CPU): model load ~7.9s once, then a short reply synthesizes in ~0.86s (within the ~1s budget); wav is valid 44.1kHz PCM. GPU (cuda) is selectable via WSAI_MELO_DEVICE for lower latency, pending GPU approval. 7 smoke tests still pass. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
110
wsai/backends/melo.py
Normal file
110
wsai/backends/melo.py
Normal file
@@ -0,0 +1,110 @@
|
||||
"""Real Korean TTS via MeloTTS, run as a persistent out-of-venv worker.
|
||||
|
||||
MeloTTS needs its own interpreter (melo311). Loading the model costs several
|
||||
seconds, so we keep one worker process alive and stream synthesis requests to
|
||||
it (see melo_worker.py for the protocol). Each `speak()` writes a wav to
|
||||
`out_dir` and hands the path to a sink (default: log it). The Discord voice
|
||||
integration later swaps the sink for "play this wav into the call".
|
||||
|
||||
Env:
|
||||
WSAI_MELO_PYTHON interpreter with melo installed
|
||||
(default: /home/claude/jarvis-tts/melo311/bin/python)
|
||||
WSAI_MELO_DEVICE cpu | cuda | auto (default cpu; cuda needs GPU approval)
|
||||
WSAI_TTS_OUT_DIR where wavs are written (default ~/.cache/wsai/tts)
|
||||
WSAI_TTS_SPEED synthesis speed multiplier (default 1.3)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Awaitable, Callable
|
||||
|
||||
from ..interfaces import Reply
|
||||
|
||||
log = logging.getLogger("wsai.tts.melo")
|
||||
|
||||
_DEFAULT_PYTHON = "/home/claude/jarvis-tts/melo311/bin/python"
|
||||
|
||||
# A sink receives the finished wav path plus the reply it voices.
|
||||
Sink = Callable[[str, Reply], Awaitable[None]]
|
||||
|
||||
|
||||
async def _log_sink(path: str, reply: Reply) -> None:
|
||||
log.info("TTS wav ready: %s (%s)", path, reply.text[:40])
|
||||
|
||||
|
||||
class MeloTTS:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
python: str | None = None,
|
||||
device: str | None = None,
|
||||
out_dir: str | None = None,
|
||||
speed: float | None = None,
|
||||
sink: Sink | None = None,
|
||||
) -> None:
|
||||
self.python = python or os.environ.get("WSAI_MELO_PYTHON", _DEFAULT_PYTHON)
|
||||
self.device = device or os.environ.get("WSAI_MELO_DEVICE", "cpu")
|
||||
self.out_dir = Path(out_dir or os.environ.get("WSAI_TTS_OUT_DIR")
|
||||
or (Path.home() / ".cache/wsai/tts"))
|
||||
self.speed = float(speed if speed is not None
|
||||
else os.environ.get("WSAI_TTS_SPEED", "1.3"))
|
||||
self.sink = sink or _log_sink
|
||||
self._proc: asyncio.subprocess.Process | None = None
|
||||
self._lock = asyncio.Lock()
|
||||
self._n = 0
|
||||
self.load_ms: int | None = None
|
||||
|
||||
async def _ensure(self) -> None:
|
||||
if self._proc is not None and self._proc.returncode is None:
|
||||
return
|
||||
self.out_dir.mkdir(parents=True, exist_ok=True)
|
||||
env = {**os.environ, "WSAI_MELO_DEVICE": self.device}
|
||||
# Run the worker module from the wsai source tree with the melo venv.
|
||||
repo_root = str(Path(__file__).resolve().parents[2])
|
||||
self._proc = await asyncio.create_subprocess_exec(
|
||||
self.python, "-m", "wsai.backends.melo_worker",
|
||||
cwd=repo_root, env=env,
|
||||
stdin=asyncio.subprocess.PIPE,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
ready = await self._proc.stdout.readline()
|
||||
info = json.loads(ready.decode())
|
||||
if not info.get("ready"):
|
||||
raise RuntimeError(f"melo worker failed to start: {info}")
|
||||
self.load_ms = info.get("ms")
|
||||
log.info("melo worker ready in %s ms on %s", self.load_ms, info.get("device"))
|
||||
|
||||
async def speak(self, reply: Reply) -> None:
|
||||
await self._ensure()
|
||||
self._n += 1
|
||||
out = str(self.out_dir / f"tts-{self._n:06d}.wav")
|
||||
req = json.dumps({"text": reply.text, "out": out, "speed": self.speed})
|
||||
s = time.monotonic()
|
||||
async with self._lock:
|
||||
assert self._proc and self._proc.stdin and self._proc.stdout
|
||||
self._proc.stdin.write((req + "\n").encode())
|
||||
await self._proc.stdin.drain()
|
||||
resp = await self._proc.stdout.readline()
|
||||
if not resp:
|
||||
raise RuntimeError("melo worker closed unexpectedly")
|
||||
res = json.loads(resp.decode())
|
||||
if not res.get("ok"):
|
||||
raise RuntimeError(f"melo synth failed: {res.get('error')}")
|
||||
log.debug("synth %d ms (worker %s ms)", int((time.monotonic() - s) * 1000), res.get("ms"))
|
||||
await self.sink(res["out"], reply)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
if self._proc is not None and self._proc.returncode is None:
|
||||
try:
|
||||
self._proc.terminate()
|
||||
await asyncio.wait_for(self._proc.wait(), timeout=5)
|
||||
except (ProcessLookupError, asyncio.TimeoutError):
|
||||
pass
|
||||
self._proc = None
|
||||
72
wsai/backends/melo_worker.py
Normal file
72
wsai/backends/melo_worker.py
Normal file
@@ -0,0 +1,72 @@
|
||||
"""Persistent MeloTTS worker (Korean).
|
||||
|
||||
MeloTTS lives in its own Python (melo311); loading the model takes seconds, so
|
||||
we load it ONCE here and then serve synthesis requests over stdin/stdout. This
|
||||
process is launched with the melo311 interpreter by wsai.backends.melo.MeloTTS.
|
||||
|
||||
MeloTTS (and its deps) print progress straight to stdout, which would corrupt
|
||||
the JSON protocol. So on startup we split the streams: a private duplicate of
|
||||
the original stdout carries the protocol, and fd 1 is redirected to fd 2 so all
|
||||
library chatter lands on stderr instead.
|
||||
|
||||
Protocol (one JSON object per line, on the protocol channel):
|
||||
<- {"text": "...", "out": "/abs/path.wav", "speed": 1.3}
|
||||
-> {"ok": true, "out": "/abs/path.wav", "ms": 123}
|
||||
-> {"ok": false, "error": "..."}
|
||||
On startup, once the model is ready, it emits exactly one line:
|
||||
-> {"ready": true, "ms": <load-ms>, "device": "cpu"}
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
# Split protocol from library noise BEFORE importing anything heavy.
|
||||
_proto = os.fdopen(os.dup(1), "w", buffering=1) # private copy of real stdout
|
||||
os.dup2(2, 1) # fd1 -> stderr, so stray library prints don't hit the protocol
|
||||
|
||||
|
||||
def _emit(obj: dict) -> None:
|
||||
_proto.write(json.dumps(obj) + "\n")
|
||||
_proto.flush()
|
||||
|
||||
|
||||
def _log(*a):
|
||||
print(*a, file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
lang = "KR"
|
||||
device = os.environ.get("WSAI_MELO_DEVICE", "cpu") # "cpu" | "cuda" | "auto"
|
||||
t0 = time.monotonic()
|
||||
from melo.api import TTS # heavy import; only in the melo venv
|
||||
|
||||
tts = TTS(language=lang, device=device)
|
||||
speaker_id = tts.hps.data.spk2id[lang]
|
||||
load_ms = int((time.monotonic() - t0) * 1000)
|
||||
_emit({"ready": True, "ms": load_ms, "device": device})
|
||||
_log(f"[melo_worker] model ready in {load_ms} ms on {device}")
|
||||
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
req = json.loads(line)
|
||||
text = req["text"]
|
||||
out = req["out"]
|
||||
speed = float(req.get("speed", 1.0))
|
||||
if out.startswith("/tmp") or out.startswith("/dev/shm"):
|
||||
raise ValueError(f"refusing RAM-backed tmpfs path: {out}")
|
||||
s = time.monotonic()
|
||||
tts.tts_to_file(text, speaker_id, out, speed=speed)
|
||||
ms = int((time.monotonic() - s) * 1000)
|
||||
_emit({"ok": True, "out": out, "ms": ms})
|
||||
except Exception as exc: # keep the worker alive across bad requests
|
||||
_emit({"ok": False, "error": f"{type(exc).__name__}: {exc}"})
|
||||
_log(f"[melo_worker] error: {exc}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user