Files
javis_bot/src/voice_worker/melo_child.py
javis-bot 27eb1315b9 fix(tts): MeloTTS speaks up to 1.25 itself so fast speech sounds natural
Stretching all of the speed-up from 1.0 kept every sound but sounded
awkward (UTMOS 3.13 at 1.5, against 3.73 for MeloTTS's own 1.5), while
MeloTTS's own speed past 1.25 drops sounds (sentence-final "요" and first
syllables, 4.6% character errors at 1.5). MeloTTS now speaks up to 1.25
and the WSOLA stretch does only the rest, with a 5 ms search: 2.6% and
UTMOS 3.56 at 1.5 through the real child. Version 0.1.6.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 00:06:52 +09:00

150 lines
6.1 KiB
Python

"""MeloTTS child process for the voice worker. Runs in its own Python 3.11 venv (transformers 4.27).
Stand-alone on purpose: it must not import the worker package, which lives in another venv.
stdin: one JSON request per line: {"text": "...", "rate": 48000, "speed": 1.5} (speed optional,
default --speed)
stdout: frames of u32 LE header length | header JSON | payload. The first frame is
{"ready": true, "rate": <native rate>}; each reply is {"ok": true, "samples": N} followed by N
int16 LE mono samples at the requested rate, or {"ok": false, "error": "..."} with no payload.
Library output that would land on stdout is sent to stderr so it cannot corrupt the frames.
Speed: MeloTTS's own speed past about 1.25 drops sounds (five Korean sentences, three renders each,
read back by Whisper large-v3: at 1.5 sentence-final "요" and first syllables went missing, 4.6%
character errors, 9.6% at 1.75, against 1.2% at 1.0 and none added at 1.25). A time stretch that keeps
the pitch (WSOLA) keeps every sound but sounds less natural the more it stretches (UTMOS 3.13 at 1.5
stretched from 1.0, against 3.73 for the model's own 1.5). So the model speaks up to MODEL_SPEED_MAX
and the stretch does only the rest: 2.6% and UTMOS 3.56 at 1.5, 1.9% and 3.09 at 1.75 (through this
child). A 5 ms search scored higher than 10 ms (UTMOS 3.42 against 3.27 for a 1.2 stretch).
"""
from __future__ import annotations
import argparse
import importlib
import json
import os
import struct
import sys
import types
MODEL_SPEED_MAX = 1.25
def split_speed(speed: float) -> tuple[float, float]:
"""(the model's speed, the stretch factor after it); their product is ``speed``."""
model = min(speed, MODEL_SPEED_MAX)
return model, speed / model
def stretch(audio, factor: float, rate: int, frame_ms: float = 30.0, search_ms: float = 5.0):
"""Shorten ``audio`` by ``factor`` and keep its pitch (WSOLA): half-overlapping Hann frames are
taken ``factor`` times further apart than they are laid down, each shifted within ``search_ms``
to where it best continues the frame before it, so the pieces join without dips or clicks."""
import numpy as np
x = np.asarray(audio, dtype=np.float64)
if abs(factor - 1.0) < 1e-3 or len(x) == 0:
return x
n = int(rate * frame_ms / 1000) // 2 * 2
hop = n // 2
tol = int(rate * search_ms / 1000)
win = 0.5 - 0.5 * np.cos(2 * np.pi * np.arange(n) / n) # its half-overlapped copies sum to 1
n_out = int(np.ceil(len(x) / factor))
frames = n_out // hop + 2
xp = np.pad(x, (tol, int(frames * hop * factor) + 2 * n + 2 * tol))
y = np.zeros(frames * hop + n)
prev = None
for k in range(frames):
pos = int(round(k * hop * factor)) + tol
if prev is not None:
natural = xp[prev + hop: prev + hop + n]
pos += int(np.argmax(np.correlate(xp[pos - tol: pos + tol + n], natural, mode="valid"))) - tol
y[k * hop: k * hop + n] += xp[pos: pos + n] * win
prev = pos
return y[:n_out]
def korean_only_patches() -> None:
"""MeloTTS imports its Japanese front end (module-level MeCab.Tagger()) even for Korean, and on
Windows g2pkk asks for ``eunjeon`` instead of python-mecab-ko. Same fix as the M0 kit."""
try:
import MeCab # noqa: F401
MeCab.Tagger # noqa: B018
except Exception:
stub = types.ModuleType("MeCab")
class Tagger:
def __init__(self, *a, **k):
pass
def parse(self, text):
raise RuntimeError("Japanese MeCab is not installed")
stub.Tagger = Tagger
sys.modules["MeCab"] = stub
try:
import g2pkk.g2pkk as g2p_mod
except Exception:
return
if g2p_mod.platform.system() == "Windows":
g2p_mod.G2p.check_mecab = lambda self: None
g2p_mod.G2p.get_mecab = lambda self: importlib.import_module("mecab").MeCab()
def main() -> int:
# Python puts this file's folder first on sys.path; drop it so the worker's own modules there
# never shadow the MeloTTS packages.
here = os.path.dirname(os.path.abspath(__file__))
sys.path[:] = [p for p in sys.path if os.path.abspath(p or ".") != here]
ap = argparse.ArgumentParser()
ap.add_argument("--language", default="KR")
ap.add_argument("--device", default="cuda")
ap.add_argument("--speed", type=float, default=1.5)
args = ap.parse_args()
out = os.fdopen(os.dup(sys.stdout.fileno()), "wb")
os.dup2(sys.stderr.fileno(), sys.stdout.fileno())
sys.stdout = sys.stderr
def reply(header: dict, payload: bytes = b"") -> None:
raw = json.dumps(header).encode()
out.write(struct.pack("<I", len(raw)) + raw + payload)
out.flush()
try:
korean_only_patches()
import numpy as np
from melo.api import TTS
from scipy.signal import resample_poly
model = TTS(language=args.language, device=args.device)
spk_map = model.hps.data.spk2id
speaker = spk_map[args.language] if args.language in spk_map else spk_map[list(spk_map.keys())[0]]
native = int(model.hps.data.sampling_rate)
model.tts_to_file("워밍업입니다.", speaker, None, speed=split_speed(args.speed)[0], quiet=True)
except Exception as e:
reply({"ready": False, "error": f"{type(e).__name__}: {e}"})
return 1
reply({"ready": True, "rate": native})
for line in sys.stdin:
try:
req = json.loads(line)
rate = int(req["rate"])
model_speed, factor = split_speed(float(req.get("speed", args.speed)))
audio = model.tts_to_file(str(req["text"]), speaker, None, speed=model_speed, quiet=True)
audio = stretch(audio, factor, native)
if rate != native:
from math import gcd
g = gcd(rate, native)
audio = resample_poly(audio, rate // g, native // g)
pcm = (np.clip(audio, -1.0, 1.0) * 32767).astype("<i2")
reply({"ok": True, "samples": int(len(pcm))}, pcm.tobytes())
except Exception as e:
reply({"ok": False, "error": f"{type(e).__name__}: {e}"})
return 0
if __name__ == "__main__":
sys.exit(main())