Stretching all of the speed-up from 1.0 kept every sound but sounded awkward (UTMOS 3.13 at 1.5, against 3.73 for MeloTTS's own 1.5), while MeloTTS's own speed past 1.25 drops sounds (sentence-final "요" and first syllables, 4.6% character errors at 1.5). MeloTTS now speaks up to 1.25 and the WSOLA stretch does only the rest, with a 5 ms search: 2.6% and UTMOS 3.56 at 1.5 through the real child. Version 0.1.6. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
150 lines
6.1 KiB
Python
150 lines
6.1 KiB
Python
"""MeloTTS child process for the voice worker. Runs in its own Python 3.11 venv (transformers 4.27).
|
|
|
|
Stand-alone on purpose: it must not import the worker package, which lives in another venv.
|
|
|
|
stdin: one JSON request per line: {"text": "...", "rate": 48000, "speed": 1.5} (speed optional,
|
|
default --speed)
|
|
stdout: frames of u32 LE header length | header JSON | payload. The first frame is
|
|
{"ready": true, "rate": <native rate>}; each reply is {"ok": true, "samples": N} followed by N
|
|
int16 LE mono samples at the requested rate, or {"ok": false, "error": "..."} with no payload.
|
|
Library output that would land on stdout is sent to stderr so it cannot corrupt the frames.
|
|
|
|
Speed: MeloTTS's own speed past about 1.25 drops sounds (five Korean sentences, three renders each,
|
|
read back by Whisper large-v3: at 1.5 sentence-final "요" and first syllables went missing, 4.6%
|
|
character errors, 9.6% at 1.75, against 1.2% at 1.0 and none added at 1.25). A time stretch that keeps
|
|
the pitch (WSOLA) keeps every sound but sounds less natural the more it stretches (UTMOS 3.13 at 1.5
|
|
stretched from 1.0, against 3.73 for the model's own 1.5). So the model speaks up to MODEL_SPEED_MAX
|
|
and the stretch does only the rest: 2.6% and UTMOS 3.56 at 1.5, 1.9% and 3.09 at 1.75 (through this
|
|
child). A 5 ms search scored higher than 10 ms (UTMOS 3.42 against 3.27 for a 1.2 stretch).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import importlib
|
|
import json
|
|
import os
|
|
import struct
|
|
import sys
|
|
import types
|
|
|
|
MODEL_SPEED_MAX = 1.25
|
|
|
|
|
|
def split_speed(speed: float) -> tuple[float, float]:
|
|
"""(the model's speed, the stretch factor after it); their product is ``speed``."""
|
|
model = min(speed, MODEL_SPEED_MAX)
|
|
return model, speed / model
|
|
|
|
|
|
def stretch(audio, factor: float, rate: int, frame_ms: float = 30.0, search_ms: float = 5.0):
|
|
"""Shorten ``audio`` by ``factor`` and keep its pitch (WSOLA): half-overlapping Hann frames are
|
|
taken ``factor`` times further apart than they are laid down, each shifted within ``search_ms``
|
|
to where it best continues the frame before it, so the pieces join without dips or clicks."""
|
|
import numpy as np
|
|
|
|
x = np.asarray(audio, dtype=np.float64)
|
|
if abs(factor - 1.0) < 1e-3 or len(x) == 0:
|
|
return x
|
|
n = int(rate * frame_ms / 1000) // 2 * 2
|
|
hop = n // 2
|
|
tol = int(rate * search_ms / 1000)
|
|
win = 0.5 - 0.5 * np.cos(2 * np.pi * np.arange(n) / n) # its half-overlapped copies sum to 1
|
|
n_out = int(np.ceil(len(x) / factor))
|
|
frames = n_out // hop + 2
|
|
xp = np.pad(x, (tol, int(frames * hop * factor) + 2 * n + 2 * tol))
|
|
y = np.zeros(frames * hop + n)
|
|
prev = None
|
|
for k in range(frames):
|
|
pos = int(round(k * hop * factor)) + tol
|
|
if prev is not None:
|
|
natural = xp[prev + hop: prev + hop + n]
|
|
pos += int(np.argmax(np.correlate(xp[pos - tol: pos + tol + n], natural, mode="valid"))) - tol
|
|
y[k * hop: k * hop + n] += xp[pos: pos + n] * win
|
|
prev = pos
|
|
return y[:n_out]
|
|
|
|
|
|
def korean_only_patches() -> None:
|
|
"""MeloTTS imports its Japanese front end (module-level MeCab.Tagger()) even for Korean, and on
|
|
Windows g2pkk asks for ``eunjeon`` instead of python-mecab-ko. Same fix as the M0 kit."""
|
|
try:
|
|
import MeCab # noqa: F401
|
|
MeCab.Tagger # noqa: B018
|
|
except Exception:
|
|
stub = types.ModuleType("MeCab")
|
|
|
|
class Tagger:
|
|
def __init__(self, *a, **k):
|
|
pass
|
|
|
|
def parse(self, text):
|
|
raise RuntimeError("Japanese MeCab is not installed")
|
|
|
|
stub.Tagger = Tagger
|
|
sys.modules["MeCab"] = stub
|
|
try:
|
|
import g2pkk.g2pkk as g2p_mod
|
|
except Exception:
|
|
return
|
|
if g2p_mod.platform.system() == "Windows":
|
|
g2p_mod.G2p.check_mecab = lambda self: None
|
|
g2p_mod.G2p.get_mecab = lambda self: importlib.import_module("mecab").MeCab()
|
|
|
|
|
|
def main() -> int:
|
|
# Python puts this file's folder first on sys.path; drop it so the worker's own modules there
|
|
# never shadow the MeloTTS packages.
|
|
here = os.path.dirname(os.path.abspath(__file__))
|
|
sys.path[:] = [p for p in sys.path if os.path.abspath(p or ".") != here]
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--language", default="KR")
|
|
ap.add_argument("--device", default="cuda")
|
|
ap.add_argument("--speed", type=float, default=1.5)
|
|
args = ap.parse_args()
|
|
|
|
out = os.fdopen(os.dup(sys.stdout.fileno()), "wb")
|
|
os.dup2(sys.stderr.fileno(), sys.stdout.fileno())
|
|
sys.stdout = sys.stderr
|
|
|
|
def reply(header: dict, payload: bytes = b"") -> None:
|
|
raw = json.dumps(header).encode()
|
|
out.write(struct.pack("<I", len(raw)) + raw + payload)
|
|
out.flush()
|
|
|
|
try:
|
|
korean_only_patches()
|
|
import numpy as np
|
|
from melo.api import TTS
|
|
from scipy.signal import resample_poly
|
|
model = TTS(language=args.language, device=args.device)
|
|
spk_map = model.hps.data.spk2id
|
|
speaker = spk_map[args.language] if args.language in spk_map else spk_map[list(spk_map.keys())[0]]
|
|
native = int(model.hps.data.sampling_rate)
|
|
model.tts_to_file("워밍업입니다.", speaker, None, speed=split_speed(args.speed)[0], quiet=True)
|
|
except Exception as e:
|
|
reply({"ready": False, "error": f"{type(e).__name__}: {e}"})
|
|
return 1
|
|
reply({"ready": True, "rate": native})
|
|
|
|
for line in sys.stdin:
|
|
try:
|
|
req = json.loads(line)
|
|
rate = int(req["rate"])
|
|
model_speed, factor = split_speed(float(req.get("speed", args.speed)))
|
|
audio = model.tts_to_file(str(req["text"]), speaker, None, speed=model_speed, quiet=True)
|
|
audio = stretch(audio, factor, native)
|
|
if rate != native:
|
|
from math import gcd
|
|
g = gcd(rate, native)
|
|
audio = resample_poly(audio, rate // g, native // g)
|
|
pcm = (np.clip(audio, -1.0, 1.0) * 32767).astype("<i2")
|
|
reply({"ok": True, "samples": int(len(pcm))}, pcm.tobytes())
|
|
except Exception as e:
|
|
reply({"ok": False, "error": f"{type(e).__name__}: {e}"})
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|