feat(voice): express [감정] tags via pitch/speed instead of speaking them

Emotion tags now steer delivery rather than being read aloud. parse_segments()
splits a reply on [감정] tags: a recognised emotion word switches the pitch and
speed of the text that follows (and is dropped), while a non-emotion bracket
(e.g. [1번]) keeps its inner words as spoken content. Emotions can change
mid-reply, so a single turn is synthesised as several pitch-shifted segments and
concatenated in the melo worker (librosa pitch_shift, warmed at startup).

The emotion vocabulary is grounded in Azure Neural TTS speaking styles plus
Ekman's basic emotions, with Korean synonyms. The brain persona is updated to
emit inline tags from that set.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
EJClaw
2026-08-22 10:14:49 +09:00
parent f585ed7b76
commit 4db73bf69f
5 changed files with 286 additions and 6 deletions

View File

@@ -11,10 +11,17 @@ library chatter lands on stderr instead.
Protocol (one JSON object per line, on the protocol channel):
<- {"text": "...", "out": "/abs/path.wav", "speed": 1.3}
<- {"segments": [{"text": "...", "speed": 1.3, "pitch": 2.0}, ...],
"out": "/abs/path.wav"} # expressive form: per-segment speed + pitch
-> {"ok": true, "out": "/abs/path.wav", "ms": 123}
-> {"ok": false, "error": "..."}
On startup, once the model is ready, it emits exactly one line:
-> {"ready": true, "ms": <load-ms>, "device": "cpu"}
``pitch`` is a semitone offset applied to that segment's wav (0 == no shift) so
emotion tags can raise/lower the voice without changing the words. Segments are
synthesised independently and concatenated with a short gap so a single reply can
carry several emotions.
"""
import json
@@ -66,8 +73,41 @@ def main() -> None:
else:
raise
speaker_id = tts.hps.data.spk2id[lang]
sr = tts.hps.data.sampling_rate
load_ms = int((time.monotonic() - t0) * 1000)
import numpy as np
import soundfile
_GAP = np.zeros(int(sr * 0.12), dtype=np.float32) # 120 ms between segments
def _pitch_shift(audio, semitones: float):
if not semitones:
return audio
import librosa
return librosa.effects.pitch_shift(
audio.astype(np.float32), sr=sr, n_steps=float(semitones)
)
def _synth_segments(segments: list[dict], out: str) -> None:
"""Synthesize each segment, pitch-shift it, and concatenate to one wav."""
pieces = []
for i, seg in enumerate(segments):
text = seg["text"]
if not text.strip():
continue
speed = float(seg.get("speed", 1.0))
pitch = float(seg.get("pitch", 0.0))
audio = tts.tts_to_file(text, speaker_id, None, speed=speed)
audio = _pitch_shift(np.asarray(audio, dtype=np.float32), pitch)
if pieces:
pieces.append(_GAP)
pieces.append(audio)
if not pieces:
raise ValueError("no speakable segment")
soundfile.write(out, np.concatenate(pieces), sr)
# Warm up before signalling ready: the first CUDA synth pays a large lazy
# cost (kernel autotune/cudnn), ~10s cold vs ~130ms hot, which would blow the
# voice loop's ~1s budget on the very first reply. Do that dummy synth here so
@@ -78,6 +118,11 @@ def main() -> None:
os.makedirs(os.path.dirname(warm_out), exist_ok=True)
w = time.monotonic()
tts.tts_to_file("워밍업", speaker_id, warm_out, speed=1.3)
# Also JIT-warm librosa's pitch shifter (first call pays ~0.4s numba
# compile) so the first *emotional* reply doesn't stall.
import librosa
librosa.effects.pitch_shift(np.zeros(sr, dtype=np.float32), sr=sr, n_steps=1.0)
warmup_ms = int((time.monotonic() - w) * 1000)
except Exception as exc:
_log(f"[melo_worker] warmup skipped: {exc}")
@@ -91,13 +136,15 @@ def main() -> None:
continue
try:
req = json.loads(line)
text = req["text"]
out = req["out"]
speed = float(req.get("speed", 1.0))
if out.startswith("/tmp") or out.startswith("/dev/shm"):
raise ValueError(f"refusing RAM-backed tmpfs path: {out}")
s = time.monotonic()
tts.tts_to_file(text, speaker_id, out, speed=speed)
if "segments" in req:
_synth_segments(req["segments"], out)
else: # legacy single-utterance form
speed = float(req.get("speed", 1.0))
tts.tts_to_file(req["text"], speaker_id, out, speed=speed)
ms = int((time.monotonic() - s) * 1000)
_emit({"ok": True, "out": out, "ms": ms})
except Exception as exc: # keep the worker alive across bad requests