feat(voice): express [감정] tags via pitch/speed instead of speaking them
Emotion tags now steer delivery rather than being read aloud. parse_segments() splits a reply on [감정] tags: a recognised emotion word switches the pitch and speed of the text that follows (and is dropped), while a non-emotion bracket (e.g. [1번]) keeps its inner words as spoken content. Emotions can change mid-reply, so a single turn is synthesised as several pitch-shifted segments and concatenated in the melo worker (librosa pitch_shift, warmed at startup). The emotion vocabulary is grounded in Azure Neural TTS speaking styles plus Ekman's basic emotions, with Korean synonyms. The brain persona is updated to emit inline tags from that set. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -11,10 +11,17 @@ library chatter lands on stderr instead.
|
||||
|
||||
Protocol (one JSON object per line, on the protocol channel):
|
||||
<- {"text": "...", "out": "/abs/path.wav", "speed": 1.3}
|
||||
<- {"segments": [{"text": "...", "speed": 1.3, "pitch": 2.0}, ...],
|
||||
"out": "/abs/path.wav"} # expressive form: per-segment speed + pitch
|
||||
-> {"ok": true, "out": "/abs/path.wav", "ms": 123}
|
||||
-> {"ok": false, "error": "..."}
|
||||
On startup, once the model is ready, it emits exactly one line:
|
||||
-> {"ready": true, "ms": <load-ms>, "device": "cpu"}
|
||||
|
||||
``pitch`` is a semitone offset applied to that segment's wav (0 == no shift) so
|
||||
emotion tags can raise/lower the voice without changing the words. Segments are
|
||||
synthesised independently and concatenated with a short gap so a single reply can
|
||||
carry several emotions.
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -66,8 +73,41 @@ def main() -> None:
|
||||
else:
|
||||
raise
|
||||
speaker_id = tts.hps.data.spk2id[lang]
|
||||
sr = tts.hps.data.sampling_rate
|
||||
load_ms = int((time.monotonic() - t0) * 1000)
|
||||
|
||||
import numpy as np
|
||||
import soundfile
|
||||
|
||||
_GAP = np.zeros(int(sr * 0.12), dtype=np.float32) # 120 ms between segments
|
||||
|
||||
def _pitch_shift(audio, semitones: float):
|
||||
if not semitones:
|
||||
return audio
|
||||
import librosa
|
||||
|
||||
return librosa.effects.pitch_shift(
|
||||
audio.astype(np.float32), sr=sr, n_steps=float(semitones)
|
||||
)
|
||||
|
||||
def _synth_segments(segments: list[dict], out: str) -> None:
|
||||
"""Synthesize each segment, pitch-shift it, and concatenate to one wav."""
|
||||
pieces = []
|
||||
for i, seg in enumerate(segments):
|
||||
text = seg["text"]
|
||||
if not text.strip():
|
||||
continue
|
||||
speed = float(seg.get("speed", 1.0))
|
||||
pitch = float(seg.get("pitch", 0.0))
|
||||
audio = tts.tts_to_file(text, speaker_id, None, speed=speed)
|
||||
audio = _pitch_shift(np.asarray(audio, dtype=np.float32), pitch)
|
||||
if pieces:
|
||||
pieces.append(_GAP)
|
||||
pieces.append(audio)
|
||||
if not pieces:
|
||||
raise ValueError("no speakable segment")
|
||||
soundfile.write(out, np.concatenate(pieces), sr)
|
||||
|
||||
# Warm up before signalling ready: the first CUDA synth pays a large lazy
|
||||
# cost (kernel autotune/cudnn), ~10s cold vs ~130ms hot, which would blow the
|
||||
# voice loop's ~1s budget on the very first reply. Do that dummy synth here so
|
||||
@@ -78,6 +118,11 @@ def main() -> None:
|
||||
os.makedirs(os.path.dirname(warm_out), exist_ok=True)
|
||||
w = time.monotonic()
|
||||
tts.tts_to_file("워밍업", speaker_id, warm_out, speed=1.3)
|
||||
# Also JIT-warm librosa's pitch shifter (first call pays ~0.4s numba
|
||||
# compile) so the first *emotional* reply doesn't stall.
|
||||
import librosa
|
||||
|
||||
librosa.effects.pitch_shift(np.zeros(sr, dtype=np.float32), sr=sr, n_steps=1.0)
|
||||
warmup_ms = int((time.monotonic() - w) * 1000)
|
||||
except Exception as exc:
|
||||
_log(f"[melo_worker] warmup skipped: {exc}")
|
||||
@@ -91,13 +136,15 @@ def main() -> None:
|
||||
continue
|
||||
try:
|
||||
req = json.loads(line)
|
||||
text = req["text"]
|
||||
out = req["out"]
|
||||
speed = float(req.get("speed", 1.0))
|
||||
if out.startswith("/tmp") or out.startswith("/dev/shm"):
|
||||
raise ValueError(f"refusing RAM-backed tmpfs path: {out}")
|
||||
s = time.monotonic()
|
||||
tts.tts_to_file(text, speaker_id, out, speed=speed)
|
||||
if "segments" in req:
|
||||
_synth_segments(req["segments"], out)
|
||||
else: # legacy single-utterance form
|
||||
speed = float(req.get("speed", 1.0))
|
||||
tts.tts_to_file(req["text"], speaker_id, out, speed=speed)
|
||||
ms = int((time.monotonic() - s) * 1000)
|
||||
_emit({"ok": True, "out": out, "ms": ms})
|
||||
except Exception as exc: # keep the worker alive across bad requests
|
||||
|
||||
Reference in New Issue
Block a user