feat(tts): real Korean TTS via persistent MeloTTS worker

Adds a MeloTTS backend that runs the model in its own melo311 interpreter
as a long-lived worker (melo_worker.py), loaded once and fed synthesis
requests over a stdin/stdout JSON protocol. fd1 is split from fd2 in the
worker so MeloTTS's stdout progress chatter can't corrupt the protocol.
Each speak() writes a wav and hands the path to a pluggable sink (the
Discord voice step will swap in "play into the call"). factory wires
tts=melo; pipeline.aclose now also tears down the tts worker.

Verified (CPU): model load ~7.9s once, then a short reply synthesizes in
~0.86s (within the ~1s budget); wav is valid 44.1kHz PCM. GPU (cuda) is
selectable via WSAI_MELO_DEVICE for lower latency, pending GPU approval.
7 smoke tests still pass.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
EJClaw
2026-08-18 18:14:44 +09:00
parent 4898192ae1
commit 6a138eff3a
5 changed files with 191 additions and 3 deletions

110
wsai/backends/melo.py Normal file
View File

@@ -0,0 +1,110 @@
"""Real Korean TTS via MeloTTS, run as a persistent out-of-venv worker.
MeloTTS needs its own interpreter (melo311). Loading the model costs several
seconds, so we keep one worker process alive and stream synthesis requests to
it (see melo_worker.py for the protocol). Each `speak()` writes a wav to
`out_dir` and hands the path to a sink (default: log it). The Discord voice
integration later swaps the sink for "play this wav into the call".
Env:
WSAI_MELO_PYTHON interpreter with melo installed
(default: /home/claude/jarvis-tts/melo311/bin/python)
WSAI_MELO_DEVICE cpu | cuda | auto (default cpu; cuda needs GPU approval)
WSAI_TTS_OUT_DIR where wavs are written (default ~/.cache/wsai/tts)
WSAI_TTS_SPEED synthesis speed multiplier (default 1.3)
"""
from __future__ import annotations
import asyncio
import json
import logging
import os
import time
from pathlib import Path
from typing import Awaitable, Callable
from ..interfaces import Reply
log = logging.getLogger("wsai.tts.melo")
_DEFAULT_PYTHON = "/home/claude/jarvis-tts/melo311/bin/python"
# A sink receives the finished wav path plus the reply it voices.
Sink = Callable[[str, Reply], Awaitable[None]]
async def _log_sink(path: str, reply: Reply) -> None:
log.info("TTS wav ready: %s (%s)", path, reply.text[:40])
class MeloTTS:
def __init__(
self,
*,
python: str | None = None,
device: str | None = None,
out_dir: str | None = None,
speed: float | None = None,
sink: Sink | None = None,
) -> None:
self.python = python or os.environ.get("WSAI_MELO_PYTHON", _DEFAULT_PYTHON)
self.device = device or os.environ.get("WSAI_MELO_DEVICE", "cpu")
self.out_dir = Path(out_dir or os.environ.get("WSAI_TTS_OUT_DIR")
or (Path.home() / ".cache/wsai/tts"))
self.speed = float(speed if speed is not None
else os.environ.get("WSAI_TTS_SPEED", "1.3"))
self.sink = sink or _log_sink
self._proc: asyncio.subprocess.Process | None = None
self._lock = asyncio.Lock()
self._n = 0
self.load_ms: int | None = None
async def _ensure(self) -> None:
if self._proc is not None and self._proc.returncode is None:
return
self.out_dir.mkdir(parents=True, exist_ok=True)
env = {**os.environ, "WSAI_MELO_DEVICE": self.device}
# Run the worker module from the wsai source tree with the melo venv.
repo_root = str(Path(__file__).resolve().parents[2])
self._proc = await asyncio.create_subprocess_exec(
self.python, "-m", "wsai.backends.melo_worker",
cwd=repo_root, env=env,
stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.DEVNULL,
)
ready = await self._proc.stdout.readline()
info = json.loads(ready.decode())
if not info.get("ready"):
raise RuntimeError(f"melo worker failed to start: {info}")
self.load_ms = info.get("ms")
log.info("melo worker ready in %s ms on %s", self.load_ms, info.get("device"))
async def speak(self, reply: Reply) -> None:
await self._ensure()
self._n += 1
out = str(self.out_dir / f"tts-{self._n:06d}.wav")
req = json.dumps({"text": reply.text, "out": out, "speed": self.speed})
s = time.monotonic()
async with self._lock:
assert self._proc and self._proc.stdin and self._proc.stdout
self._proc.stdin.write((req + "\n").encode())
await self._proc.stdin.drain()
resp = await self._proc.stdout.readline()
if not resp:
raise RuntimeError("melo worker closed unexpectedly")
res = json.loads(resp.decode())
if not res.get("ok"):
raise RuntimeError(f"melo synth failed: {res.get('error')}")
log.debug("synth %d ms (worker %s ms)", int((time.monotonic() - s) * 1000), res.get("ms"))
await self.sink(res["out"], reply)
async def aclose(self) -> None:
if self._proc is not None and self._proc.returncode is None:
try:
self._proc.terminate()
await asyncio.wait_for(self._proc.wait(), timeout=5)
except (ProcessLookupError, asyncio.TimeoutError):
pass
self._proc = None