Hotword occurrences were counted with fuzzy matching, which never counted a one-letter wake word (so a hinted "봇 ..." was accepted over a plain decode without it) and counted the real word "서비스" as "자비스" (so the hint could turn it into the wake word). Occurrences are now counted exactly (case, spacing and punctuation ignored); near misses are left as heard for call detection. The real-noise eval now plays each recording as recorded and at -25 dBFS and reports how many clips reached the model. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
319 lines
14 KiB
Python
319 lines
14 KiB
Python
"""Voice worker with the real models: ten recorded clips streamed at once, and a TTS round trip.
|
|
|
|
Opt-in because it needs a CUDA GPU and the downloaded models:
|
|
|
|
JARVIS_VOICE_REAL_TESTS=1 [JARVIS_MELO_PYTHON=<MeloTTS venv python>] pytest tests/test_voice_worker_real.py -s
|
|
|
|
On Linux, put the pip CUDA libraries on LD_LIBRARY_PATH (nvidia/cublas/lib, nvidia/cudnn/lib).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import re
|
|
import time
|
|
import wave
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
pytestmark = [pytest.mark.integration,
|
|
pytest.mark.skipif(os.environ.get("JARVIS_VOICE_REAL_TESTS") != "1",
|
|
reason="set JARVIS_VOICE_REAL_TESTS=1 to run on the GPU")]
|
|
|
|
SAMPLES = Path(__file__).resolve().parent.parent / "scripts" / "m0" / "assets" / "ko_samples"
|
|
TOKEN = "real-test-token"
|
|
FRAME_MS = 20
|
|
|
|
|
|
def cer(ref: str, hyp: str) -> float:
|
|
a, b = (re.sub(r"[\W_]+", "", s) for s in (ref, hyp))
|
|
prev = list(range(len(b) + 1))
|
|
for i, ca in enumerate(a, 1):
|
|
cur = [i]
|
|
for j, cb in enumerate(b, 1):
|
|
cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb)))
|
|
prev = cur
|
|
return prev[-1] / max(1, len(a))
|
|
|
|
|
|
def load(name: str) -> np.ndarray:
|
|
with wave.open(str(SAMPLES / name), "rb") as w:
|
|
assert w.getframerate() == 16000 and w.getnchannels() == 1
|
|
return np.frombuffer(w.readframes(w.getnframes()), dtype="<i2").astype(np.float32) / 32768.0
|
|
|
|
|
|
def to_48k_stereo(x16: np.ndarray) -> np.ndarray:
|
|
t48 = np.arange(len(x16) * 3) / 3
|
|
x48 = np.interp(t48, np.arange(len(x16)), x16)
|
|
return np.repeat(x48, 2)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def engines():
|
|
from voice_worker.stt_whisper import WhisperTranscriber
|
|
from voice_worker.vad import SileroVad
|
|
stt = WhisperTranscriber()
|
|
stt.warm_up()
|
|
tts = None
|
|
if os.environ.get("JARVIS_MELO_PYTHON"):
|
|
from voice_worker.tts_melo import MeloEngine
|
|
tts = MeloEngine(os.environ["JARVIS_MELO_PYTHON"])
|
|
yield stt, SileroVad(), tts
|
|
if tts:
|
|
tts.close()
|
|
|
|
|
|
async def with_worker(engines, body):
|
|
from websockets.asyncio.client import connect
|
|
from voice_worker.server import VoiceWorker
|
|
stt, vad, tts = engines
|
|
worker = VoiceWorker(TOKEN, stt, vad.stream, tts)
|
|
server = await worker.serve("127.0.0.1", 0)
|
|
url = f"ws://127.0.0.1:{server.sockets[0].getsockname()[1]}"
|
|
try:
|
|
async with connect(url, max_size=2 ** 24) as ws:
|
|
await ws.send(json.dumps({"type": "hello", "token": TOKEN, "stt": {"hotwords": ["자비스"]}}))
|
|
assert json.loads(await ws.recv())["type"] == "hello_ok"
|
|
return await asyncio.wait_for(body(ws), 120)
|
|
finally:
|
|
server.close()
|
|
await server.wait_closed()
|
|
await worker.stop()
|
|
|
|
|
|
def test_ten_speakers_at_once(engines):
|
|
"""Ten speakers talk over each other in real time (48 kHz stereo, 20 ms frames, staggered starts).
|
|
Half end with Discord's speaking_end, half keep sending silence so only the VAD can end them."""
|
|
from voice_worker.protocol import encode_audio_frame
|
|
|
|
refs = json.loads((SAMPLES / "refs.json").read_text(encoding="utf-8"))["samples"]
|
|
files = [refs[i % len(refs)] for i in range(10)]
|
|
speakers = [f"spk{i}" for i in range(10)]
|
|
frame = 48000 * FRAME_MS // 1000 * 2 # stereo samples per frame
|
|
|
|
def track(i: int) -> list[bytes]:
|
|
lead = np.zeros(int(0.15 * i * 16000), dtype=np.float32)
|
|
tail = np.zeros(int(0.6 * 16000), dtype=np.float32) if i % 2 else np.zeros(0, dtype=np.float32)
|
|
clip = load(files[i]["file"])
|
|
if i % 2 == 0: # these end with speaking_end: Discord stops sending right after the last word
|
|
loud = np.nonzero(np.abs(clip) > 0.02)[0]
|
|
clip = clip[: loud[-1] + 800]
|
|
x = to_48k_stereo(np.concatenate([lead, clip, tail]))
|
|
pcm = (np.clip(x, -1, 1) * 32767).astype("<i2")
|
|
return [pcm[k:k + frame].tobytes() for k in range(0, len(pcm), frame)]
|
|
|
|
tracks = [track(i) for i in range(10)]
|
|
|
|
async def body(ws):
|
|
finals: dict[str, list[dict]] = {s: [] for s in speakers}
|
|
partials = 0
|
|
finished = asyncio.Event()
|
|
|
|
async def reader():
|
|
nonlocal partials
|
|
async for raw in ws:
|
|
msg = json.loads(raw)
|
|
if msg["type"] == "speech_start":
|
|
open_utts[msg["speaker"]] = msg["utt"]
|
|
elif msg["type"] == "partial":
|
|
partials += 1
|
|
elif msg["type"] == "final":
|
|
finals[msg["speaker"]].append(msg)
|
|
if sent_all.is_set() and all(finals.values()) and closed_all():
|
|
finished.set()
|
|
|
|
open_utts: dict[str, int] = {}
|
|
|
|
def closed_all() -> bool:
|
|
return all(len(finals[s]) >= open_utts.get(s, 0) for s in speakers)
|
|
|
|
sent_all = asyncio.Event()
|
|
rd = asyncio.create_task(reader())
|
|
t0 = time.monotonic()
|
|
for k in range(max(len(t) for t in tracks)):
|
|
for i, s in enumerate(speakers):
|
|
if k < len(tracks[i]):
|
|
await ws.send(encode_audio_frame(s, tracks[i][k]))
|
|
elif k == len(tracks[i]) and i % 2 == 0:
|
|
await ws.send(json.dumps({"type": "speaking_end", "speaker": s}))
|
|
await asyncio.sleep(max(0.0, t0 + (k + 1) * FRAME_MS / 1000 - time.monotonic()))
|
|
sent_all.set()
|
|
try:
|
|
await asyncio.wait_for(finished.wait(), 20)
|
|
except asyncio.TimeoutError:
|
|
pass
|
|
rd.cancel()
|
|
return finals, partials
|
|
|
|
finals, partials = asyncio.run(with_worker(engines, body))
|
|
rows = []
|
|
for i, s in enumerate(speakers):
|
|
text = " ".join(m["text"] for m in finals[s]).strip()
|
|
rows.append((s, files[i]["file"], len(finals[s]), cer(files[i]["text"], text),
|
|
[m["end_reason"] for m in finals[s]], [m["wait_ms"] for m in finals[s]], text))
|
|
for r in rows:
|
|
print(*r)
|
|
waits = [w for r in rows for w in r[5]]
|
|
print(f"partials={partials} wait_ms median={np.median(waits):.0f} max={max(waits)}")
|
|
assert all(r[2] >= 1 for r in rows), "every speaker gets a final"
|
|
assert np.mean([r[3] for r in rows]) <= 0.15
|
|
assert max(r[3] for r in rows) <= 0.35
|
|
assert {"speaking_end", "silence"} <= {reason for r in rows for reason in r[4]}
|
|
assert partials > 0
|
|
|
|
|
|
def test_tts_reply_comes_back_as_the_same_words(engines):
|
|
"""Speech the worker synthesises, fed back in as a speaker, is transcribed to the same text."""
|
|
if engines[2] is None:
|
|
pytest.skip("set JARVIS_MELO_PYTHON to the MeloTTS venv python")
|
|
from voice_worker.protocol import decode_tts_frame, encode_audio_frame
|
|
# MeloTTS samples its voice differently each time and blurs some words ("기온은 이십삼" came back
|
|
# as "비오는 이시찬" in raw, peak-normalised and equalised audio alike), so the sentence is one it
|
|
# says clearly; this checks the worker's plumbing, not MeloTTS's diction.
|
|
text = "오늘 서울은 맑고 따뜻해요. 우산은 집에 두고 가셔도 돼요."
|
|
|
|
async def body(ws):
|
|
await ws.send(json.dumps({"type": "config", "partials": False}))
|
|
await ws.send(json.dumps({"type": "tts", "id": "t1", "text": text}))
|
|
chunks = []
|
|
t0 = time.monotonic()
|
|
first = None
|
|
while True:
|
|
raw = await ws.recv()
|
|
if isinstance(raw, bytes):
|
|
chunks.append(decode_tts_frame(raw))
|
|
first = first or time.monotonic() - t0
|
|
elif json.loads(raw)["type"] == "tts_done":
|
|
break
|
|
pcm = b"".join(c[2] for c in chunks)
|
|
step = 48000 * FRAME_MS // 1000 * 2 * 2
|
|
for k in range(0, len(pcm), step):
|
|
await ws.send(encode_audio_frame("echo", pcm[k:k + step]))
|
|
await ws.send(json.dumps({"type": "speaking_end", "speaker": "echo"}))
|
|
got = []
|
|
# The pause between the sentences may split them, and the clip's own trailing silence may end
|
|
# the last one before speaking_end does, so collect finals until the worker goes quiet.
|
|
while True:
|
|
try:
|
|
msg = json.loads(await asyncio.wait_for(ws.recv(), 3))
|
|
except asyncio.TimeoutError:
|
|
break
|
|
if msg["type"] == "final":
|
|
got.append(msg["text"])
|
|
return chunks, first, " ".join(got)
|
|
|
|
chunks, first, heard = asyncio.run(with_worker(engines, body))
|
|
print(f"tts chunks={len(chunks)} first_chunk_s={first:.2f} heard={heard}")
|
|
assert [c[1] for c in chunks] == [0, 1]
|
|
assert cer(text, heard) <= 0.2
|
|
|
|
|
|
def _stream_clip_16k(engines, clip: np.ndarray, stt: dict) -> list[dict]:
|
|
"""Stream one 16 kHz mono clip as a single speaker (as fast as the socket takes it), end it,
|
|
and return every final the worker sends."""
|
|
from voice_worker.protocol import encode_audio_frame
|
|
pcm = (np.clip(to_48k_stereo(clip), -1, 1) * 32767).astype("<i2").tobytes()
|
|
step = 48000 * FRAME_MS // 1000 * 2 * 2
|
|
|
|
async def body(ws):
|
|
await ws.send(json.dumps({"type": "config", **stt}))
|
|
for k in range(0, len(pcm), step):
|
|
await ws.send(encode_audio_frame("s", pcm[k:k + step]))
|
|
await ws.send(json.dumps({"type": "speaking_end", "speaker": "s"}))
|
|
finals = []
|
|
while True:
|
|
try:
|
|
msg = json.loads(await asyncio.wait_for(ws.recv(), 3))
|
|
except asyncio.TimeoutError:
|
|
return finals
|
|
if msg["type"] == "final":
|
|
finals.append(msg)
|
|
|
|
return asyncio.run(with_worker(engines, body))
|
|
|
|
|
|
def _babble(seconds: float, rng) -> np.ndarray:
|
|
"""Speech-like noise: the kit's Korean clips played backwards and overlaid (voice-like, no words)."""
|
|
n = int(16000 * seconds)
|
|
x = np.zeros(n, dtype=np.float32)
|
|
for name in ("ko_01.wav", "ko_03.wav", "ko_05.wav", "ko_07.wav"):
|
|
c = load(name)[::-1]
|
|
x += np.roll(np.tile(c, n // len(c) + 1)[:n], int(rng.integers(0, n)))
|
|
return x
|
|
|
|
|
|
def _at_db(x: np.ndarray, db: float) -> np.ndarray:
|
|
return (x * 10 ** (db / 20) / (np.sqrt(np.mean(x.astype(np.float64) ** 2)) + 1e-12)).astype(np.float32)
|
|
|
|
|
|
@pytest.mark.parametrize("level_db", [-40, -25])
|
|
def test_voice_like_noise_with_the_wake_word_hint_produces_no_text(engines, level_db):
|
|
"""Backwards babble passes the VAD. Given the wake word as a hint, Whisper used to write the wake
|
|
word (a phantom call to the bot); without words in the audio no final may carry text."""
|
|
rng = np.random.default_rng(level_db + 100)
|
|
finals = _stream_clip_16k(engines, _at_db(_babble(6.0, rng), level_db),
|
|
{"hotwords": ["자비스"], "partials": False})
|
|
print(level_db, [(m["text"], m["end_reason"]) for m in finals])
|
|
assert all(m["text"] == "" for m in finals)
|
|
|
|
|
|
@pytest.mark.parametrize("snr_db", [20, 10])
|
|
def test_speech_over_voice_like_noise_keeps_the_words(engines, snr_db):
|
|
rng = np.random.default_rng(snr_db)
|
|
refs = json.loads((SAMPLES / "refs.json").read_text(encoding="utf-8"))["samples"]
|
|
errs = []
|
|
for r in refs[:4]:
|
|
speech = np.concatenate([np.zeros(8000, np.float32), _at_db(load(r["file"]), -25), np.zeros(8000, np.float32)])
|
|
noisy = speech + _at_db(_babble(len(speech) / 16000, rng), -25 - snr_db)
|
|
finals = _stream_clip_16k(engines, noisy, {"hotwords": ["자비스"], "partials": False})
|
|
text = " ".join(m["text"] for m in finals).strip()
|
|
errs.append(cer(r["text"], text))
|
|
print(snr_db, r["file"], round(errs[-1], 2), text)
|
|
assert np.mean(errs) <= 0.15
|
|
|
|
|
|
@pytest.mark.parametrize("level_db", [-60, -45, -10])
|
|
def test_quiet_and_loud_speakers_are_both_understood(engines, level_db):
|
|
refs = json.loads((SAMPLES / "refs.json").read_text(encoding="utf-8"))["samples"]
|
|
errs = []
|
|
for r in refs[:4]:
|
|
x = np.concatenate([np.zeros(8000, np.float32), _at_db(load(r["file"]), level_db), np.zeros(8000, np.float32)])
|
|
if level_db == -10:
|
|
x = np.clip(x * 4, -1, 1) # shouting into a hot mic: clipped
|
|
finals = _stream_clip_16k(engines, x, {"partials": False})
|
|
errs.append(cer(r["text"], " ".join(m["text"] for m in finals)))
|
|
print(level_db, [round(e, 2) for e in errs])
|
|
assert np.mean(errs) <= 0.15
|
|
|
|
|
|
def test_real_noise_recordings_produce_no_text(engines):
|
|
"""Optional: point JARVIS_NOISE_DIR at noise-only recordings (16 kHz or 44.1 kHz mono/stereo WAV,
|
|
e.g. ESC-50 keyboard, mouse, breathing, coughing, laughing; ESC-50 is CC BY-NC, so it is not in
|
|
the repository). Each plays as recorded and at -25 dBFS; none may produce text."""
|
|
folder = os.environ.get("JARVIS_NOISE_DIR")
|
|
if not folder:
|
|
pytest.skip("set JARVIS_NOISE_DIR to a folder of noise-only WAV files")
|
|
from scipy.signal import resample_poly
|
|
texts, reached, clips = [], 0, 0
|
|
for path in sorted(Path(folder).glob("*.wav")):
|
|
with wave.open(str(path), "rb") as w:
|
|
x = np.frombuffer(w.readframes(w.getnframes()), dtype="<i2").astype(np.float32) / 32768.0
|
|
if w.getnchannels() == 2:
|
|
x = x.reshape(-1, 2).mean(axis=1)
|
|
rate = w.getframerate()
|
|
if rate != 16000:
|
|
from math import gcd
|
|
g = gcd(rate, 16000)
|
|
x = resample_poly(x, 16000 // g, rate // g).astype(np.float32)
|
|
for level in (x, np.clip(_at_db(x, -25), -1, 1)): # as recorded, and as a loud mic would send it
|
|
clips += 1
|
|
finals = _stream_clip_16k(engines, level, {"hotwords": ["자비스"], "partials": False})
|
|
reached += bool(finals) # a final means the clip passed the VAD and went to the model
|
|
texts += [(path.name, m["text"]) for m in finals if m["text"]]
|
|
print(f"noise clips {clips}, reached the model {reached}, with text {texts}")
|
|
assert texts == []
|