"""Voice worker with the real models: ten recorded clips streamed at once, and a TTS round trip. Opt-in because it needs a CUDA GPU and the downloaded models: JARVIS_VOICE_REAL_TESTS=1 [JARVIS_MELO_PYTHON=] pytest tests/test_voice_worker_real.py -s On Linux, put the pip CUDA libraries on LD_LIBRARY_PATH (nvidia/cublas/lib, nvidia/cudnn/lib). """ from __future__ import annotations import asyncio import json import os import re import time import wave from pathlib import Path import numpy as np import pytest pytestmark = [pytest.mark.integration, pytest.mark.skipif(os.environ.get("JARVIS_VOICE_REAL_TESTS") != "1", reason="set JARVIS_VOICE_REAL_TESTS=1 to run on the GPU")] SAMPLES = Path(__file__).resolve().parent.parent / "scripts" / "m0" / "assets" / "ko_samples" TOKEN = "real-test-token" FRAME_MS = 20 def cer(ref: str, hyp: str) -> float: a, b = (re.sub(r"[\W_]+", "", s) for s in (ref, hyp)) prev = list(range(len(b) + 1)) for i, ca in enumerate(a, 1): cur = [i] for j, cb in enumerate(b, 1): cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb))) prev = cur return prev[-1] / max(1, len(a)) def load(name: str) -> np.ndarray: with wave.open(str(SAMPLES / name), "rb") as w: assert w.getframerate() == 16000 and w.getnchannels() == 1 return np.frombuffer(w.readframes(w.getnframes()), dtype=" np.ndarray: t48 = np.arange(len(x16) * 3) / 3 x48 = np.interp(t48, np.arange(len(x16)), x16) return np.repeat(x48, 2) @pytest.fixture(scope="module") def engines(): from voice_worker.stt_whisper import WhisperTranscriber from voice_worker.vad import SileroVad stt = WhisperTranscriber() stt.warm_up() tts = None if os.environ.get("JARVIS_MELO_PYTHON"): from voice_worker.tts_melo import MeloEngine tts = MeloEngine(os.environ["JARVIS_MELO_PYTHON"]) yield stt, SileroVad(), tts if tts: tts.close() async def with_worker(engines, body): from websockets.asyncio.client import connect from voice_worker.server import VoiceWorker stt, vad, tts = engines worker = VoiceWorker(TOKEN, stt, vad.stream, tts) server = await worker.serve("127.0.0.1", 0) url = f"ws://127.0.0.1:{server.sockets[0].getsockname()[1]}" try: async with connect(url, max_size=2 ** 24) as ws: await ws.send(json.dumps({"type": "hello", "token": TOKEN, "stt": {"hotwords": ["자비스"]}})) assert json.loads(await ws.recv())["type"] == "hello_ok" return await asyncio.wait_for(body(ws), 120) finally: server.close() await server.wait_closed() await worker.stop() def test_ten_speakers_at_once(engines): """Ten speakers talk over each other in real time (48 kHz stereo, 20 ms frames, staggered starts). Half end with Discord's speaking_end, half keep sending silence so only the VAD can end them.""" from voice_worker.protocol import encode_audio_frame refs = json.loads((SAMPLES / "refs.json").read_text(encoding="utf-8"))["samples"] files = [refs[i % len(refs)] for i in range(10)] speakers = [f"spk{i}" for i in range(10)] frame = 48000 * FRAME_MS // 1000 * 2 # stereo samples per frame def track(i: int) -> list[bytes]: lead = np.zeros(int(0.15 * i * 16000), dtype=np.float32) tail = np.zeros(int(0.6 * 16000), dtype=np.float32) if i % 2 else np.zeros(0, dtype=np.float32) clip = load(files[i]["file"]) if i % 2 == 0: # these end with speaking_end: Discord stops sending right after the last word loud = np.nonzero(np.abs(clip) > 0.02)[0] clip = clip[: loud[-1] + 800] x = to_48k_stereo(np.concatenate([lead, clip, tail])) pcm = (np.clip(x, -1, 1) * 32767).astype(" bool: return all(len(finals[s]) >= open_utts.get(s, 0) for s in speakers) sent_all = asyncio.Event() rd = asyncio.create_task(reader()) t0 = time.monotonic() for k in range(max(len(t) for t in tracks)): for i, s in enumerate(speakers): if k < len(tracks[i]): await ws.send(encode_audio_frame(s, tracks[i][k])) elif k == len(tracks[i]) and i % 2 == 0: await ws.send(json.dumps({"type": "speaking_end", "speaker": s})) await asyncio.sleep(max(0.0, t0 + (k + 1) * FRAME_MS / 1000 - time.monotonic())) sent_all.set() try: await asyncio.wait_for(finished.wait(), 20) except asyncio.TimeoutError: pass rd.cancel() return finals, partials finals, partials = asyncio.run(with_worker(engines, body)) rows = [] for i, s in enumerate(speakers): text = " ".join(m["text"] for m in finals[s]).strip() rows.append((s, files[i]["file"], len(finals[s]), cer(files[i]["text"], text), [m["end_reason"] for m in finals[s]], [m["wait_ms"] for m in finals[s]], text)) for r in rows: print(*r) waits = [w for r in rows for w in r[5]] print(f"partials={partials} wait_ms median={np.median(waits):.0f} max={max(waits)}") assert all(r[2] >= 1 for r in rows), "every speaker gets a final" assert np.mean([r[3] for r in rows]) <= 0.15 assert max(r[3] for r in rows) <= 0.35 assert {"speaking_end", "silence"} <= {reason for r in rows for reason in r[4]} assert partials > 0 def test_tts_reply_comes_back_as_the_same_words(engines): """Speech the worker synthesises, fed back in as a speaker, is transcribed to the same text.""" if engines[2] is None: pytest.skip("set JARVIS_MELO_PYTHON to the MeloTTS venv python") from voice_worker.protocol import decode_tts_frame, encode_audio_frame # MeloTTS samples its voice differently each time and blurs some words ("기온은 이십삼" came back # as "비오는 이시찬" in raw, peak-normalised and equalised audio alike), so the sentence is one it # says clearly; this checks the worker's plumbing, not MeloTTS's diction. text = "오늘 서울은 맑고 따뜻해요. 우산은 집에 두고 가셔도 돼요." async def body(ws): await ws.send(json.dumps({"type": "config", "partials": False})) await ws.send(json.dumps({"type": "tts", "id": "t1", "text": text})) chunks = [] t0 = time.monotonic() first = None while True: raw = await ws.recv() if isinstance(raw, bytes): chunks.append(decode_tts_frame(raw)) first = first or time.monotonic() - t0 elif json.loads(raw)["type"] == "tts_done": break pcm = b"".join(c[2] for c in chunks) step = 48000 * FRAME_MS // 1000 * 2 * 2 for k in range(0, len(pcm), step): await ws.send(encode_audio_frame("echo", pcm[k:k + step])) await ws.send(json.dumps({"type": "speaking_end", "speaker": "echo"})) got = [] # The pause between the sentences may split them, and the clip's own trailing silence may end # the last one before speaking_end does, so collect finals until the worker goes quiet. while True: try: msg = json.loads(await asyncio.wait_for(ws.recv(), 3)) except asyncio.TimeoutError: break if msg["type"] == "final": got.append(msg["text"]) return chunks, first, " ".join(got) chunks, first, heard = asyncio.run(with_worker(engines, body)) print(f"tts chunks={len(chunks)} first_chunk_s={first:.2f} heard={heard}") assert [c[1] for c in chunks] == [0, 1] assert cer(text, heard) <= 0.2 def _stream_clip_16k(engines, clip: np.ndarray, stt: dict) -> list[dict]: """Stream one 16 kHz mono clip as a single speaker (as fast as the socket takes it), end it, and return every final the worker sends.""" from voice_worker.protocol import encode_audio_frame pcm = (np.clip(to_48k_stereo(clip), -1, 1) * 32767).astype(" np.ndarray: """Speech-like noise: the kit's Korean clips played backwards and overlaid (voice-like, no words).""" n = int(16000 * seconds) x = np.zeros(n, dtype=np.float32) for name in ("ko_01.wav", "ko_03.wav", "ko_05.wav", "ko_07.wav"): c = load(name)[::-1] x += np.roll(np.tile(c, n // len(c) + 1)[:n], int(rng.integers(0, n))) return x def _at_db(x: np.ndarray, db: float) -> np.ndarray: return (x * 10 ** (db / 20) / (np.sqrt(np.mean(x.astype(np.float64) ** 2)) + 1e-12)).astype(np.float32) @pytest.mark.parametrize("level_db", [-40, -25]) def test_voice_like_noise_with_the_wake_word_hint_produces_no_text(engines, level_db): """Backwards babble passes the VAD. Given the wake word as a hint, Whisper used to write the wake word (a phantom call to the bot); without words in the audio no final may carry text.""" rng = np.random.default_rng(level_db + 100) finals = _stream_clip_16k(engines, _at_db(_babble(6.0, rng), level_db), {"hotwords": ["자비스"], "partials": False}) print(level_db, [(m["text"], m["end_reason"]) for m in finals]) assert all(m["text"] == "" for m in finals) @pytest.mark.parametrize("snr_db", [20, 10]) def test_speech_over_voice_like_noise_keeps_the_words(engines, snr_db): rng = np.random.default_rng(snr_db) refs = json.loads((SAMPLES / "refs.json").read_text(encoding="utf-8"))["samples"] errs = [] for r in refs[:4]: speech = np.concatenate([np.zeros(8000, np.float32), _at_db(load(r["file"]), -25), np.zeros(8000, np.float32)]) noisy = speech + _at_db(_babble(len(speech) / 16000, rng), -25 - snr_db) finals = _stream_clip_16k(engines, noisy, {"hotwords": ["자비스"], "partials": False}) text = " ".join(m["text"] for m in finals).strip() errs.append(cer(r["text"], text)) print(snr_db, r["file"], round(errs[-1], 2), text) assert np.mean(errs) <= 0.15 @pytest.mark.parametrize("level_db", [-60, -45, -10]) def test_quiet_and_loud_speakers_are_both_understood(engines, level_db): refs = json.loads((SAMPLES / "refs.json").read_text(encoding="utf-8"))["samples"] errs = [] for r in refs[:4]: x = np.concatenate([np.zeros(8000, np.float32), _at_db(load(r["file"]), level_db), np.zeros(8000, np.float32)]) if level_db == -10: x = np.clip(x * 4, -1, 1) # shouting into a hot mic: clipped finals = _stream_clip_16k(engines, x, {"partials": False}) errs.append(cer(r["text"], " ".join(m["text"] for m in finals))) print(level_db, [round(e, 2) for e in errs]) assert np.mean(errs) <= 0.15 def test_real_noise_recordings_produce_no_text(engines): """Optional: point JARVIS_NOISE_DIR at noise-only recordings (16 kHz or 44.1 kHz mono/stereo WAV, e.g. ESC-50 keyboard, mouse, breathing, coughing, laughing; ESC-50 is CC BY-NC, so it is not in the repository). Each plays as recorded and at -25 dBFS; none may produce text.""" folder = os.environ.get("JARVIS_NOISE_DIR") if not folder: pytest.skip("set JARVIS_NOISE_DIR to a folder of noise-only WAV files") from scipy.signal import resample_poly texts, reached, clips = [], 0, 0 for path in sorted(Path(folder).glob("*.wav")): with wave.open(str(path), "rb") as w: x = np.frombuffer(w.readframes(w.getnframes()), dtype="