From 6a138eff3a0cd630b49d582b13484844c63d9d60 Mon Sep 17 00:00:00 2001 From: EJClaw Date: Tue, 18 Aug 2026 18:14:44 +0900 Subject: [PATCH] feat(tts): real Korean TTS via persistent MeloTTS worker Adds a MeloTTS backend that runs the model in its own melo311 interpreter as a long-lived worker (melo_worker.py), loaded once and fed synthesis requests over a stdin/stdout JSON protocol. fd1 is split from fd2 in the worker so MeloTTS's stdout progress chatter can't corrupt the protocol. Each speak() writes a wav and hands the path to a pluggable sink (the Discord voice step will swap in "play into the call"). factory wires tts=melo; pipeline.aclose now also tears down the tts worker. Verified (CPU): model load ~7.9s once, then a short reply synthesizes in ~0.86s (within the ~1s budget); wav is valid 44.1kHz PCM. GPU (cuda) is selectable via WSAI_MELO_DEVICE for lower latency, pending GPU approval. 7 smoke tests still pass. Co-Authored-By: Claude Opus 4.7 --- wsai/backends/melo.py | 110 +++++++++++++++++++++++++++++++++++ wsai/backends/melo_worker.py | 72 +++++++++++++++++++++++ wsai/config.py | 2 +- wsai/factory.py | 4 ++ wsai/pipeline.py | 6 +- 5 files changed, 191 insertions(+), 3 deletions(-) create mode 100644 wsai/backends/melo.py create mode 100644 wsai/backends/melo_worker.py diff --git a/wsai/backends/melo.py b/wsai/backends/melo.py new file mode 100644 index 0000000..632f7b1 --- /dev/null +++ b/wsai/backends/melo.py @@ -0,0 +1,110 @@ +"""Real Korean TTS via MeloTTS, run as a persistent out-of-venv worker. + +MeloTTS needs its own interpreter (melo311). Loading the model costs several +seconds, so we keep one worker process alive and stream synthesis requests to +it (see melo_worker.py for the protocol). Each `speak()` writes a wav to +`out_dir` and hands the path to a sink (default: log it). The Discord voice +integration later swaps the sink for "play this wav into the call". + +Env: + WSAI_MELO_PYTHON interpreter with melo installed + (default: /home/claude/jarvis-tts/melo311/bin/python) + WSAI_MELO_DEVICE cpu | cuda | auto (default cpu; cuda needs GPU approval) + WSAI_TTS_OUT_DIR where wavs are written (default ~/.cache/wsai/tts) + WSAI_TTS_SPEED synthesis speed multiplier (default 1.3) +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import os +import time +from pathlib import Path +from typing import Awaitable, Callable + +from ..interfaces import Reply + +log = logging.getLogger("wsai.tts.melo") + +_DEFAULT_PYTHON = "/home/claude/jarvis-tts/melo311/bin/python" + +# A sink receives the finished wav path plus the reply it voices. +Sink = Callable[[str, Reply], Awaitable[None]] + + +async def _log_sink(path: str, reply: Reply) -> None: + log.info("TTS wav ready: %s (%s)", path, reply.text[:40]) + + +class MeloTTS: + def __init__( + self, + *, + python: str | None = None, + device: str | None = None, + out_dir: str | None = None, + speed: float | None = None, + sink: Sink | None = None, + ) -> None: + self.python = python or os.environ.get("WSAI_MELO_PYTHON", _DEFAULT_PYTHON) + self.device = device or os.environ.get("WSAI_MELO_DEVICE", "cpu") + self.out_dir = Path(out_dir or os.environ.get("WSAI_TTS_OUT_DIR") + or (Path.home() / ".cache/wsai/tts")) + self.speed = float(speed if speed is not None + else os.environ.get("WSAI_TTS_SPEED", "1.3")) + self.sink = sink or _log_sink + self._proc: asyncio.subprocess.Process | None = None + self._lock = asyncio.Lock() + self._n = 0 + self.load_ms: int | None = None + + async def _ensure(self) -> None: + if self._proc is not None and self._proc.returncode is None: + return + self.out_dir.mkdir(parents=True, exist_ok=True) + env = {**os.environ, "WSAI_MELO_DEVICE": self.device} + # Run the worker module from the wsai source tree with the melo venv. + repo_root = str(Path(__file__).resolve().parents[2]) + self._proc = await asyncio.create_subprocess_exec( + self.python, "-m", "wsai.backends.melo_worker", + cwd=repo_root, env=env, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + ready = await self._proc.stdout.readline() + info = json.loads(ready.decode()) + if not info.get("ready"): + raise RuntimeError(f"melo worker failed to start: {info}") + self.load_ms = info.get("ms") + log.info("melo worker ready in %s ms on %s", self.load_ms, info.get("device")) + + async def speak(self, reply: Reply) -> None: + await self._ensure() + self._n += 1 + out = str(self.out_dir / f"tts-{self._n:06d}.wav") + req = json.dumps({"text": reply.text, "out": out, "speed": self.speed}) + s = time.monotonic() + async with self._lock: + assert self._proc and self._proc.stdin and self._proc.stdout + self._proc.stdin.write((req + "\n").encode()) + await self._proc.stdin.drain() + resp = await self._proc.stdout.readline() + if not resp: + raise RuntimeError("melo worker closed unexpectedly") + res = json.loads(resp.decode()) + if not res.get("ok"): + raise RuntimeError(f"melo synth failed: {res.get('error')}") + log.debug("synth %d ms (worker %s ms)", int((time.monotonic() - s) * 1000), res.get("ms")) + await self.sink(res["out"], reply) + + async def aclose(self) -> None: + if self._proc is not None and self._proc.returncode is None: + try: + self._proc.terminate() + await asyncio.wait_for(self._proc.wait(), timeout=5) + except (ProcessLookupError, asyncio.TimeoutError): + pass + self._proc = None diff --git a/wsai/backends/melo_worker.py b/wsai/backends/melo_worker.py new file mode 100644 index 0000000..b0c0d64 --- /dev/null +++ b/wsai/backends/melo_worker.py @@ -0,0 +1,72 @@ +"""Persistent MeloTTS worker (Korean). + +MeloTTS lives in its own Python (melo311); loading the model takes seconds, so +we load it ONCE here and then serve synthesis requests over stdin/stdout. This +process is launched with the melo311 interpreter by wsai.backends.melo.MeloTTS. + +MeloTTS (and its deps) print progress straight to stdout, which would corrupt +the JSON protocol. So on startup we split the streams: a private duplicate of +the original stdout carries the protocol, and fd 1 is redirected to fd 2 so all +library chatter lands on stderr instead. + +Protocol (one JSON object per line, on the protocol channel): + <- {"text": "...", "out": "/abs/path.wav", "speed": 1.3} + -> {"ok": true, "out": "/abs/path.wav", "ms": 123} + -> {"ok": false, "error": "..."} +On startup, once the model is ready, it emits exactly one line: + -> {"ready": true, "ms": , "device": "cpu"} +""" + +import json +import os +import sys +import time + +# Split protocol from library noise BEFORE importing anything heavy. +_proto = os.fdopen(os.dup(1), "w", buffering=1) # private copy of real stdout +os.dup2(2, 1) # fd1 -> stderr, so stray library prints don't hit the protocol + + +def _emit(obj: dict) -> None: + _proto.write(json.dumps(obj) + "\n") + _proto.flush() + + +def _log(*a): + print(*a, file=sys.stderr, flush=True) + + +def main() -> None: + lang = "KR" + device = os.environ.get("WSAI_MELO_DEVICE", "cpu") # "cpu" | "cuda" | "auto" + t0 = time.monotonic() + from melo.api import TTS # heavy import; only in the melo venv + + tts = TTS(language=lang, device=device) + speaker_id = tts.hps.data.spk2id[lang] + load_ms = int((time.monotonic() - t0) * 1000) + _emit({"ready": True, "ms": load_ms, "device": device}) + _log(f"[melo_worker] model ready in {load_ms} ms on {device}") + + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + req = json.loads(line) + text = req["text"] + out = req["out"] + speed = float(req.get("speed", 1.0)) + if out.startswith("/tmp") or out.startswith("/dev/shm"): + raise ValueError(f"refusing RAM-backed tmpfs path: {out}") + s = time.monotonic() + tts.tts_to_file(text, speaker_id, out, speed=speed) + ms = int((time.monotonic() - s) * 1000) + _emit({"ok": True, "out": out, "ms": ms}) + except Exception as exc: # keep the worker alive across bad requests + _emit({"ok": False, "error": f"{type(exc).__name__}: {exc}"}) + _log(f"[melo_worker] error: {exc}") + + +if __name__ == "__main__": + main() diff --git a/wsai/config.py b/wsai/config.py index 858b1be..19e7369 100644 --- a/wsai/config.py +++ b/wsai/config.py @@ -19,7 +19,7 @@ class Settings: source: str | None = "mock" # mock | mss | None (eyes-free) vision: str | None = "mock" # mock | claude | None (eyes-free) stt: str | None = "mock" # mock | (whisper) | None - tts: str | None = "mock" # mock | (melo) | None + tts: str | None = "mock" # mock | melo | None brain: str = "mock" # mock | claude text: str | None = None # None | (discord) diff --git a/wsai/factory.py b/wsai/factory.py index 5c268d4..ca322d0 100644 --- a/wsai/factory.py +++ b/wsai/factory.py @@ -77,6 +77,10 @@ def _stt(s: Settings): def _tts(s: Settings): if s.tts in (None, "none"): return None + if s.tts == "melo": + from .backends.melo import MeloTTS + + return MeloTTS() from .backends.mock import MockTTS return MockTTS() diff --git a/wsai/pipeline.py b/wsai/pipeline.py index cc8841c..cca3943 100644 --- a/wsai/pipeline.py +++ b/wsai/pipeline.py @@ -162,8 +162,10 @@ class Pipeline: await self.aclose() async def aclose(self) -> None: - for closer in (self.source, self.stt, self.text_channel): - if closer is not None: + # tts is included because a real TTS (e.g. MeloTTS) owns a worker + # subprocess that must be torn down; mock backends have no aclose. + for closer in (self.source, self.stt, self.text_channel, self.tts): + if closer is not None and hasattr(closer, "aclose"): try: await closer.aclose() except Exception: