0b92284ff8
Ports the 4 independent voice controls documented in tts_site's manual into the MeloTTS backend, layered on top of the existing emotion-tag segments: - glyph speed: generation-stage length_scale (existing speed path); default bumped 1.2 -> 1.25 to match the manual (still well below the 1.5 that slurred) - word_gap (sec, -0.2..0.5): scale intra-sentence silences after natural synth - sentence_gap (sec, -0.5..1.5): insert/trim silence at sentence boundaries, replacing the old fixed 120ms inter-segment gap - pitch (semitones, -12..12): global offset added on top of per-emotion pitch Each reply is split into sentences, synthesised per-sentence at its segment's speed, word_gap applied, joined with sentence_gap, then pitch-shifted. Exposed via WSAI_TTS_SPEED/WORD_GAP/SENTENCE_GAP/PITCH env vars and MeloTTS ctor args. Verified by real synthesis: each control independently changes wav duration (speed, sentence_gap, word_gap-with-pauses, pitch); 29 tests pass. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
266 lines
12 KiB
Python
266 lines
12 KiB
Python
"""Real Korean TTS via MeloTTS, run as a persistent out-of-venv worker.
|
|
|
|
MeloTTS needs its own interpreter (melo311). Loading the model costs several
|
|
seconds, so we keep one worker process alive and stream synthesis requests to
|
|
it (see melo_worker.py for the protocol). Each `speak()` writes a wav to
|
|
`out_dir` and hands the path to a sink (default: log it). The Discord voice
|
|
integration later swaps the sink for "play this wav into the call".
|
|
|
|
Env:
|
|
WSAI_MELO_PYTHON interpreter with melo installed
|
|
(default: /home/claude/jarvis-tts/melo311/bin/python)
|
|
WSAI_MELO_DEVICE cpu | cuda | auto (default auto: GPU if torch sees one,
|
|
else CPU; the worker falls back to CPU if CUDA fails)
|
|
WSAI_TTS_OUT_DIR where wavs are written (default ~/.cache/wsai/tts)
|
|
WSAI_TTS_SPEED glyph speed / length_scale multiplier (0.5..2.0, default 1.25)
|
|
WSAI_TTS_WORD_GAP intra-sentence pause delta, sec (-0.2..0.5, default 0.25)
|
|
WSAI_TTS_SENTENCE_GAP sentence-boundary silence, sec (-0.5..1.5, default 0.75)
|
|
WSAI_TTS_PITCH global semitone offset (-12..12, default 0.0)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import collections
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Awaitable, Callable
|
|
|
|
from ..interfaces import Reply
|
|
from .emotion import parse_segments
|
|
|
|
log = logging.getLogger("wsai.tts.melo")
|
|
|
|
_DEFAULT_PYTHON = "/home/claude/jarvis-tts/melo311/bin/python"
|
|
|
|
_FENCE_RE = re.compile(r"```[^\n`]*\n?(.*?)```", re.DOTALL)
|
|
_LINK_RE = re.compile(r"\[([^\]]+)\]\([^)]*\)")
|
|
_INLINE_CODE_RE = re.compile(r"`+([^`]*)`+")
|
|
|
|
|
|
def normalize_for_speech(text: str) -> str:
|
|
"""Flatten Claude's markdown/code formatting into plain prose before TTS.
|
|
|
|
MeloTTS's Korean text normaliser has no dictionary entry for characters
|
|
like the backtick and dies with ``KeyError: '`'`` — which crashes the whole
|
|
voice turn the moment the model mentions a command or shows a code block.
|
|
Code/markdown also reads terribly aloud. So strip the formatting and keep
|
|
the words. Every spoken path (dashboard voice turn and the Discord speak()
|
|
bridge) funnels through ``synth()``, so normalising there covers them both.
|
|
"""
|
|
if not text:
|
|
return text
|
|
# Fenced code block -> keep its inner text as spoken words, drop the fences.
|
|
text = _FENCE_RE.sub(lambda m: " " + m.group(1) + " ", text)
|
|
# [label](url) -> label
|
|
text = _LINK_RE.sub(r"\1", text)
|
|
# `code` -> code
|
|
text = _INLINE_CODE_RE.sub(r"\1", text)
|
|
# Any stray/unbalanced backtick that survived -> gone. This is the exact
|
|
# character that crashes MeloTTS, so guarantee none remain.
|
|
text = text.replace("`", "")
|
|
# Markdown structure markers -> plain text.
|
|
text = re.sub(r"(?m)^\s{0,3}#{1,6}\s*", "", text) # ATX headings
|
|
text = re.sub(r"(?m)^\s{0,3}>\s?", "", text) # blockquotes
|
|
text = re.sub(r"(?m)^\s{0,3}[-*+]\s+", "", text) # bullet list markers
|
|
text = re.sub(r"[*_]{1,3}", "", text) # bold/italic emphasis
|
|
# Collapse the whitespace the stripping leaves behind.
|
|
text = re.sub(r"[ \t]+", " ", text)
|
|
text = re.sub(r"\n{2,}", "\n", text)
|
|
return text.strip()
|
|
|
|
# A sink receives the finished wav path plus the reply it voices.
|
|
Sink = Callable[[str, Reply], Awaitable[None]]
|
|
|
|
|
|
async def _log_sink(path: str, reply: Reply) -> None:
|
|
log.info("TTS wav ready: %s (%s)", path, reply.text[:40])
|
|
|
|
|
|
class MeloTTS:
|
|
def __init__(
|
|
self,
|
|
*,
|
|
python: str | None = None,
|
|
device: str | None = None,
|
|
out_dir: str | None = None,
|
|
speed: float | None = None,
|
|
word_gap: float | None = None,
|
|
sentence_gap: float | None = None,
|
|
pitch: float | None = None,
|
|
sink: Sink | None = None,
|
|
) -> None:
|
|
self.python = python or os.environ.get("WSAI_MELO_PYTHON", _DEFAULT_PYTHON)
|
|
self.device = device or os.environ.get("WSAI_MELO_DEVICE", "auto")
|
|
self.out_dir = Path(out_dir or os.environ.get("WSAI_TTS_OUT_DIR")
|
|
or (Path.home() / ".cache/wsai/tts"))
|
|
self.speed = float(speed if speed is not None
|
|
else os.environ.get("WSAI_TTS_SPEED", "1.25"))
|
|
# Reply-global rhythm/pitch controls (see melo_worker.py / docs manual).
|
|
self.word_gap = float(word_gap if word_gap is not None
|
|
else os.environ.get("WSAI_TTS_WORD_GAP", "0.25"))
|
|
self.sentence_gap = float(sentence_gap if sentence_gap is not None
|
|
else os.environ.get("WSAI_TTS_SENTENCE_GAP", "0.75"))
|
|
self.pitch = float(pitch if pitch is not None
|
|
else os.environ.get("WSAI_TTS_PITCH", "0.0"))
|
|
self.sink = sink or _log_sink
|
|
self._proc: asyncio.subprocess.Process | None = None
|
|
self._lock = asyncio.Lock()
|
|
# Serialises worker (re)start + the ready handshake so a caller that
|
|
# arrives mid-warmup waits for readiness instead of reading the same
|
|
# stdout StreamReader concurrently (asyncio forbids overlapping reads).
|
|
self._start_lock = asyncio.Lock()
|
|
self._ready = False # True only after the ready handshake completes
|
|
self._n = 0
|
|
self.load_ms: int | None = None
|
|
# Keep the worker's most recent stderr lines so a crash reports its real
|
|
# cause instead of a bare JSONDecodeError. Bounded so it can't grow.
|
|
self._stderr_tail: collections.deque[str] = collections.deque(maxlen=40)
|
|
self._stderr_task: asyncio.Task | None = None
|
|
|
|
async def _drain_stderr(self, stream: asyncio.StreamReader) -> None:
|
|
# The worker redirects fd1 -> fd2, so ALL library chatter lands on
|
|
# stderr. If we PIPE stderr but never read it, the OS pipe buffer fills
|
|
# and the worker blocks forever. So we continuously drain it and keep
|
|
# only the last few lines for diagnostics.
|
|
try:
|
|
while True:
|
|
line = await stream.readline()
|
|
if not line:
|
|
return
|
|
self._stderr_tail.append(line.decode(errors="replace").rstrip())
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception: # draining must never crash the caller
|
|
return
|
|
|
|
def _stderr_hint(self) -> str:
|
|
tail = "\n".join(self._stderr_tail)
|
|
return f" worker stderr tail:\n{tail}" if tail else " (worker produced no stderr)"
|
|
|
|
async def warmup(self) -> None:
|
|
"""Start and load the worker now so the first real utterance is warm.
|
|
|
|
Called at pipeline startup so users don't wait ~7s (CPU model load) for
|
|
the very first spoken reply."""
|
|
await self._ensure()
|
|
|
|
async def _ensure(self) -> None:
|
|
# Fast path: only skip when the worker is not just spawned but fully
|
|
# handshaked. Checking `_proc` alone would let a caller sail past while
|
|
# another coroutine (e.g. warmup) is still awaiting the ready line on
|
|
# this same stdout, causing overlapping StreamReader reads.
|
|
if self._proc is not None and self._proc.returncode is None and self._ready:
|
|
return
|
|
async with self._start_lock:
|
|
# Re-check under the lock: another coroutine may have finished the
|
|
# (re)start + handshake while we waited.
|
|
if self._proc is not None and self._proc.returncode is None and self._ready:
|
|
return
|
|
self._ready = False
|
|
self.out_dir.mkdir(parents=True, exist_ok=True)
|
|
env = {**os.environ, "WSAI_MELO_DEVICE": self.device}
|
|
# Run the worker module from the wsai source tree with the melo venv.
|
|
repo_root = str(Path(__file__).resolve().parents[2])
|
|
self._proc = await asyncio.create_subprocess_exec(
|
|
self.python, "-m", "wsai.backends.melo_worker",
|
|
cwd=repo_root, env=env,
|
|
stdin=asyncio.subprocess.PIPE,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
self._stderr_tail.clear()
|
|
assert self._proc.stderr is not None
|
|
self._stderr_task = asyncio.create_task(self._drain_stderr(self._proc.stderr))
|
|
ready = await self._proc.stdout.readline()
|
|
if not ready: # worker died before signalling ready
|
|
await self._proc.wait()
|
|
raise RuntimeError(
|
|
f"melo worker exited before ready (code {self._proc.returncode})."
|
|
f"{self._stderr_hint()}"
|
|
)
|
|
try:
|
|
info = json.loads(ready.decode())
|
|
except json.JSONDecodeError as exc:
|
|
raise RuntimeError(
|
|
f"melo worker sent invalid ready line {ready!r}: {exc}."
|
|
f"{self._stderr_hint()}"
|
|
) from exc
|
|
if not info.get("ready"):
|
|
raise RuntimeError(
|
|
f"melo worker failed to start: {info}.{self._stderr_hint()}"
|
|
)
|
|
self.load_ms = info.get("ms")
|
|
self._ready = True
|
|
log.info("melo worker ready in %s ms on %s", self.load_ms, info.get("device"))
|
|
|
|
async def synth(self, text: str) -> str:
|
|
"""Synthesize `text` to a wav and return its path (no sink). Reusable by
|
|
callers that want the wav directly (e.g. the Discord voice bridge)."""
|
|
await self._ensure()
|
|
text = normalize_for_speech(text)
|
|
# Split on [감정] tags: each tag steers pitch/speed for the text that
|
|
# follows (and is itself not spoken); non-emotion brackets stay as words.
|
|
segments = parse_segments(text, self.speed)
|
|
self._n += 1
|
|
out = str(self.out_dir / f"tts-{self._n:06d}.wav")
|
|
if segments:
|
|
payload = {
|
|
"segments": [
|
|
{"text": s.text, "speed": s.speed, "pitch": s.pitch}
|
|
for s in segments
|
|
],
|
|
"out": out,
|
|
"word_gap": self.word_gap,
|
|
"sentence_gap": self.sentence_gap,
|
|
"pitch": self.pitch,
|
|
}
|
|
else: # empty/whitespace reply: keep legacy single-utterance behaviour
|
|
payload = {
|
|
"text": text,
|
|
"out": out,
|
|
"speed": self.speed,
|
|
"word_gap": self.word_gap,
|
|
"sentence_gap": self.sentence_gap,
|
|
"pitch": self.pitch,
|
|
}
|
|
req = json.dumps(payload)
|
|
s = time.monotonic()
|
|
async with self._lock:
|
|
assert self._proc and self._proc.stdin and self._proc.stdout
|
|
self._proc.stdin.write((req + "\n").encode())
|
|
await self._proc.stdin.drain()
|
|
resp = await self._proc.stdout.readline()
|
|
if not resp:
|
|
raise RuntimeError(f"melo worker closed unexpectedly.{self._stderr_hint()}")
|
|
res = json.loads(resp.decode())
|
|
if not res.get("ok"):
|
|
raise RuntimeError(f"melo synth failed: {res.get('error')}")
|
|
log.debug("synth %d ms (worker %s ms)", int((time.monotonic() - s) * 1000), res.get("ms"))
|
|
return res["out"]
|
|
|
|
async def speak(self, reply: Reply) -> None:
|
|
out = await self.synth(reply.text)
|
|
await self.sink(out, reply)
|
|
|
|
async def aclose(self) -> None:
|
|
if self._proc is not None and self._proc.returncode is None:
|
|
try:
|
|
self._proc.terminate()
|
|
await asyncio.wait_for(self._proc.wait(), timeout=5)
|
|
except (ProcessLookupError, asyncio.TimeoutError):
|
|
pass
|
|
if self._stderr_task is not None:
|
|
self._stderr_task.cancel()
|
|
try:
|
|
await self._stderr_task
|
|
except (asyncio.CancelledError, Exception):
|
|
pass
|
|
self._stderr_task = None
|
|
self._proc = None
|
|
self._ready = False
|