- 로그 screen: "보이는 글 txt로 저장" (logs_save_text), a 대화 view that saves the days' conversation with every trace in full (history_export), and a 목소리 view with the voice.record switch and the days' recordings as a zip with an index (voice_export). - Voice worker: with JARVIS_RECORDINGS_DIR (voice.record, off by default) each final and each spoken answer is kept as a WAV with an index line; day folders older than 7 days removed. - response.interrupt (default on): off, latest/important finish the running answer first. - Bot browser: Chrome lets pages start video and sound; botBrowser reports each page's media state, waits for a player to start after open/click, and presses play on a video page a click led to when its player stayed paused (live: a clicked YouTube video stayed paused). - webSearch reads `query` when `search_query` is missing (live: both searches of an answer failed). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
449 lines
18 KiB
Python
449 lines
18 KiB
Python
"""Voice worker end to end over a real local WebSocket, with fake STT/VAD/TTS engines.
|
|
|
|
The fake VAD reads speech probability off the amplitude and the fake transcriber names a clip by its
|
|
"level": each speaker sends a tone whose amplitude is the level (what the fake VAD reads) and whose
|
|
pitch is level x 1000 Hz (what the fake transcriber reads, since the worker evens out loudness).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import time
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from voice_worker.protocol import decode_tts_frame, encode_audio_frame
|
|
from voice_worker.server import CLOSE_BAD_HELLO, CLOSE_BAD_TOKEN, VoiceWorker
|
|
|
|
websockets = pytest.importorskip("websockets")
|
|
from websockets.asyncio.client import connect # noqa: E402
|
|
|
|
TOKEN = "test-token"
|
|
|
|
|
|
class AmplitudeVad:
|
|
def prob(self, window):
|
|
return float(np.max(np.abs(window)))
|
|
|
|
|
|
class LevelTranscriber:
|
|
def __init__(self):
|
|
self.calls = []
|
|
|
|
def transcribe(self, clips, opts):
|
|
self.calls.append((opts, len(clips)))
|
|
return [f"level {np.argmax(np.abs(np.fft.rfft(c))) * 16000 / len(c) / 1000:.1f}" for c in clips]
|
|
|
|
|
|
class FakeTts:
|
|
"""10 samples per character; optional delay per sentence; can fail on a marker."""
|
|
|
|
def __init__(self, delay: float = 0.0):
|
|
self.delay = delay
|
|
self.texts = []
|
|
self.speed = 1.5
|
|
self.speeds = []
|
|
|
|
def synthesize(self, text, rate):
|
|
if self.delay:
|
|
time.sleep(self.delay)
|
|
if "FAIL" in text:
|
|
raise RuntimeError("engine broke")
|
|
self.texts.append(text)
|
|
self.speeds.append(self.speed)
|
|
return np.full(len(text) * 10, 1000, dtype=np.int16)
|
|
|
|
|
|
def pcm(level: float, ms: int, rate=16000, channels=1) -> bytes:
|
|
n = rate * ms // 1000
|
|
# 20 ms frames hold whole cycles at these pitches, so frames join without a click.
|
|
x = level * np.sin(2 * np.pi * level * 1000 * np.arange(n) / rate)
|
|
return (np.repeat(x, channels) * 32767).astype("<i2").tobytes()
|
|
|
|
|
|
async def with_worker(body, transcriber=None, tts=None, **kw):
|
|
worker = VoiceWorker(TOKEN, transcriber if transcriber is not None else LevelTranscriber(), AmplitudeVad,
|
|
tts, **kw)
|
|
server = await worker.serve("127.0.0.1", 0)
|
|
port = server.sockets[0].getsockname()[1]
|
|
try:
|
|
return await asyncio.wait_for(body(f"ws://127.0.0.1:{port}"), 15)
|
|
finally:
|
|
server.close()
|
|
await server.wait_closed()
|
|
await worker.stop()
|
|
|
|
|
|
async def hello(ws, **extra):
|
|
await ws.send(json.dumps({"type": "hello", "token": TOKEN, "audio": {"rate": 16000, "channels": 1},
|
|
"tts": {"rate": 24000, "channels": 2}, **extra}))
|
|
return json.loads(await ws.recv())
|
|
|
|
|
|
async def recv_until(ws, pred, limit=200):
|
|
seen = []
|
|
for _ in range(limit):
|
|
raw = await ws.recv()
|
|
msg = raw if isinstance(raw, bytes) else json.loads(raw)
|
|
seen.append(msg)
|
|
if pred(msg):
|
|
return seen
|
|
raise AssertionError("condition never met")
|
|
|
|
|
|
async def stream(ws, speaker: str, level: float, ms: int, realtime: bool = False):
|
|
for _ in range(ms // 20):
|
|
await ws.send(encode_audio_frame(speaker, pcm(level, 20)))
|
|
if realtime:
|
|
await asyncio.sleep(0.02)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_wrong_token_is_refused():
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await ws.send(json.dumps({"type": "hello", "token": "nope"}))
|
|
with pytest.raises(websockets.ConnectionClosed) as e:
|
|
await ws.recv()
|
|
return e.value.rcvd.code
|
|
|
|
assert asyncio.run(with_worker(body)) == CLOSE_BAD_TOKEN
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_silent_client_is_dropped_after_the_hello_timeout():
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
with pytest.raises(websockets.ConnectionClosed) as e:
|
|
await ws.recv()
|
|
return e.value.rcvd.code
|
|
|
|
assert asyncio.run(with_worker(body, hello_timeout_s=0.2)) == CLOSE_BAD_HELLO
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_non_loopback_addresses_are_refused():
|
|
worker = VoiceWorker(TOKEN, LevelTranscriber(), AmplitudeVad)
|
|
with pytest.raises(ValueError):
|
|
asyncio.run(worker.serve("0.0.0.0", 0))
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_hello_ok_reports_readiness_and_effective_settings():
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
return await hello(ws, stt={"hotwords": ["자비스"], "end_silence_ms": 250})
|
|
|
|
ok = asyncio.run(with_worker(body))
|
|
assert ok["type"] == "hello_ok"
|
|
assert ok["stt_ready"] is True and ok["tts_ready"] is False
|
|
assert ok["stt"]["hotwords"] == ["자비스"] and ok["stt"]["end_silence_ms"] == 250
|
|
assert ok["tts"] == {"rate": 24000, "channels": 2}
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_two_speakers_each_get_their_own_final():
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws, stt={"partials": False})
|
|
await stream(ws, "alice", 0.9, 600)
|
|
await stream(ws, "bob", 0.6, 600)
|
|
await ws.send(json.dumps({"type": "speaking_end", "speaker": "alice"}))
|
|
await ws.send(json.dumps({"type": "speaking_end", "speaker": "bob"}))
|
|
seen = await recv_until(ws, lambda m: m.get("type") == "final" and m["speaker"] == "bob")
|
|
if not any(m.get("type") == "final" and m["speaker"] == "alice" for m in seen):
|
|
seen += await recv_until(ws, lambda m: m.get("type") == "final")
|
|
return seen
|
|
|
|
seen = asyncio.run(with_worker(body))
|
|
finals = {m["speaker"]: m for m in seen if m.get("type") == "final"}
|
|
assert finals["alice"]["text"] == "level 0.9" and finals["bob"]["text"] == "level 0.6"
|
|
assert {m["end_reason"] for m in finals.values()} == {"speaking_end"}
|
|
assert all(m["utt"] == 1 and 500 <= m["audio_ms"] <= 900 for m in finals.values())
|
|
starts = [m for m in seen if m.get("type") == "speech_start"]
|
|
assert sorted(m["speaker"] for m in starts) == ["alice", "bob"]
|
|
assert not any(m.get("type") == "partial" for m in seen)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_audio_that_stops_without_speaking_end_still_ends():
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws, stt={"partials": False, "end_silence_ms": 200})
|
|
await stream(ws, "alice", 0.9, 500)
|
|
return await recv_until(ws, lambda m: m.get("type") == "final")
|
|
|
|
final = asyncio.run(with_worker(body))[-1]
|
|
assert final["end_reason"] == "idle" and final["text"] == "level 0.9"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_partials_flow_while_speaking_and_stop_when_turned_off():
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws, stt={"partial_interval_ms": 100})
|
|
await stream(ws, "alice", 0.9, 800, realtime=True)
|
|
first = await recv_until(ws, lambda m: m.get("type") == "partial")
|
|
await ws.send(json.dumps({"type": "config", "partials": False}))
|
|
# Drain whatever was already on its way before the setting changed.
|
|
while True:
|
|
try:
|
|
await asyncio.wait_for(ws.recv(), 0.3)
|
|
except asyncio.TimeoutError:
|
|
break
|
|
await stream(ws, "alice", 0.9, 600, realtime=True)
|
|
await ws.send(json.dumps({"type": "speaking_end", "speaker": "alice"}))
|
|
rest = await recv_until(ws, lambda m: m.get("type") == "final")
|
|
return first, rest
|
|
|
|
first, rest = asyncio.run(with_worker(body))
|
|
assert first[-1]["text"] == "level 0.9" and first[-1]["utt"] == 1
|
|
assert not any(m.get("type") == "partial" for m in rest)
|
|
assert rest[-1]["type"] == "final"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_tts_streams_one_chunk_per_sentence_in_the_session_format():
|
|
tts = FakeTts()
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await ws.send(json.dumps({"type": "tts", "id": "r1", "text": "오늘은 맑아요. 기온은 17.5도예요."}))
|
|
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
|
|
|
|
seen = asyncio.run(with_worker(body, tts=tts))
|
|
chunks = [decode_tts_frame(m) for m in seen if isinstance(m, bytes)]
|
|
assert [(c[0], c[1]) for c in chunks] == [("r1", 0), ("r1", 1)]
|
|
assert tts.texts == ["오늘은 맑아요.", "기온은 17.5도예요."]
|
|
# Stereo: every mono sample twice, 16-bit.
|
|
assert len(chunks[0][2]) == len("오늘은 맑아요.") * 10 * 2 * 2
|
|
assert seen[-1] == {"type": "tts_done", "id": "r1", "chunks": 2, "cancelled": False, "failed": False}
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_tts_text_can_arrive_in_pieces():
|
|
tts = FakeTts()
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
for piece, more in [("첫 번째 문장", True), ("이에요. 두 번", True), ("째 문장이에요.", False)]:
|
|
await ws.send(json.dumps({"type": "tts", "id": "s", "text": piece, "more": more}))
|
|
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
|
|
|
|
seen = asyncio.run(with_worker(body, tts=tts))
|
|
assert tts.texts == ["첫 번째 문장이에요.", "두 번째 문장이에요."]
|
|
assert seen[-1]["chunks"] == 2
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_cancel_stops_the_remaining_sentences():
|
|
tts = FakeTts(delay=0.15)
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
text = " ".join(f"이것은 {i}번째 문장입니다." for i in range(6))
|
|
await ws.send(json.dumps({"type": "tts", "id": "c", "text": text}))
|
|
first = await recv_until(ws, lambda m: isinstance(m, bytes))
|
|
await ws.send(json.dumps({"type": "tts_cancel", "id": "c"}))
|
|
return first + await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
|
|
|
|
seen = asyncio.run(with_worker(body, tts=tts))
|
|
done = seen[-1]
|
|
assert done["cancelled"] is True and done["chunks"] < 6
|
|
assert sum(isinstance(m, bytes) for m in seen) == done["chunks"]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_failing_sentence_ends_the_request_as_failed():
|
|
tts = FakeTts()
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await ws.send(json.dumps({"type": "tts", "id": "f", "text": "첫 문장입니다. FAIL 문장입니다. 셋째 문장입니다."}))
|
|
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
|
|
|
|
seen = asyncio.run(with_worker(body, tts=tts))
|
|
errors = [m for m in seen if isinstance(m, dict) and m.get("type") == "error"]
|
|
assert errors and errors[0]["code"] == "tts_failed" and errors[0]["id"] == "f"
|
|
assert seen[-1]["failed"] is True and seen[-1]["chunks"] == 1
|
|
assert tts.texts == ["첫 문장입니다."]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_new_speech_speed_applies_to_the_sentences_made_after_it():
|
|
tts = FakeTts()
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await ws.send(json.dumps({"type": "tts", "id": "a", "text": "보통 속도 문장이에요."}))
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
|
|
await ws.send(json.dumps({"type": "tts_speed", "speed": 1.8}))
|
|
out = []
|
|
for bad in (2.5, 0.2, "fast", True):
|
|
await ws.send(json.dumps({"type": "tts_speed", "speed": bad}))
|
|
out.append(json.loads(await ws.recv()))
|
|
await ws.send(json.dumps({"type": "tts", "id": "b", "text": "빨라진 문장이에요."}))
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
|
|
return out
|
|
|
|
errors = asyncio.run(with_worker(body, tts=tts))
|
|
assert [e["code"] for e in errors] == ["bad_message"] * 4
|
|
assert tts.speeds == [1.5, 1.8], "a refused speed leaves the last good one"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_tts_without_an_engine_reports_unavailable():
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await ws.send(json.dumps({"type": "tts", "id": "x", "text": "안녕하세요."}))
|
|
return json.loads(await ws.recv())
|
|
|
|
msg = asyncio.run(with_worker(body))
|
|
assert msg["type"] == "error" and msg["code"] == "tts_unavailable" and msg["id"] == "x"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_bad_messages_get_an_error_and_the_connection_stays_open():
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
out = []
|
|
for raw in ["{not json", json.dumps({"type": "dance"}), json.dumps({"type": "config", "final_beam": 99})]:
|
|
await ws.send(raw)
|
|
out.append(json.loads(await ws.recv()))
|
|
await ws.send(b"\x01\x00")
|
|
out.append(json.loads(await ws.recv()))
|
|
await ws.send(json.dumps({"type": "config", "final_beam": 3}))
|
|
await ws.send(json.dumps({"type": "dance"}))
|
|
out.append(json.loads(await ws.recv()))
|
|
return out
|
|
|
|
out = asyncio.run(with_worker(body))
|
|
assert [m["code"] for m in out] == ["bad_message", "bad_message", "bad_message", "bad_frame", "bad_message"]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_48k_stereo_input_reaches_the_model_at_16k():
|
|
t = LevelTranscriber()
|
|
lengths = []
|
|
orig = t.transcribe
|
|
|
|
def spy(clips, opts):
|
|
lengths.extend(len(c) for c in clips)
|
|
return orig(clips, opts)
|
|
|
|
t.transcribe = spy
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await ws.send(json.dumps({"type": "hello", "token": TOKEN, "stt": {"partials": False}}))
|
|
await ws.recv()
|
|
for _ in range(30): # 600 ms
|
|
await ws.send(encode_audio_frame("a", pcm(0.9, 20, rate=48000, channels=2)))
|
|
await ws.send(json.dumps({"type": "speaking_end", "speaker": "a"}))
|
|
return await recv_until(ws, lambda m: m.get("type") == "final")
|
|
|
|
final = asyncio.run(with_worker(body, transcriber=t))[-1]
|
|
assert final["text"] == "level 0.9"
|
|
assert 0.5 * 16000 <= lengths[0] <= 0.9 * 16000
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_speakers_who_leave_free_their_place_and_finish_their_utterance():
|
|
from voice_worker.server import MAX_SPEAKERS
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws, stt={"partials": False})
|
|
seen = []
|
|
for i in range(MAX_SPEAKERS + 8):
|
|
await stream(ws, f"user{i}", 0.9, 400)
|
|
# Leaving mid-utterance still delivers that utterance's final.
|
|
await ws.send(json.dumps({"type": "speaker_leave", "speaker": f"user{i}"}))
|
|
seen += await recv_until(ws, lambda m: m.get("type") in ("final", "error"))
|
|
return seen
|
|
|
|
seen = asyncio.run(with_worker(body))
|
|
assert not [m for m in seen if m.get("type") == "error"]
|
|
finals = [m for m in seen if m.get("type") == "final"]
|
|
assert [m["speaker"] for m in finals] == [f"user{i}" for i in range(len(finals))]
|
|
assert len(finals) > MAX_SPEAKERS and all(m["text"] == "level 0.9" for m in finals)
|
|
|
|
|
|
def _index(root):
|
|
import pathlib
|
|
days = [d for d in pathlib.Path(root).iterdir() if d.is_dir()]
|
|
assert len(days) == 1
|
|
lines = [json.loads(x) for x in (days[0] / "index.jsonl").read_text(encoding="utf-8").splitlines()]
|
|
return days[0], lines
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_with_recordings_on_each_utterance_and_spoken_answer_is_kept_as_a_wav_with_its_text(tmp_path):
|
|
import wave
|
|
from voice_worker.recordings import Recorder
|
|
|
|
recorder = Recorder(tmp_path)
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws, stt={"partials": False})
|
|
await ws.send(json.dumps({"type": "member_join", "user": "alice", "name": "Alice"}))
|
|
await stream(ws, "alice", 0.9, 600)
|
|
await ws.send(json.dumps({"type": "speaking_end", "speaker": "alice"}))
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final")
|
|
await ws.send(json.dumps({"type": "tts", "id": "r1", "text": "오늘은 맑아요. 기온은 17.5도예요."}))
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
|
|
|
|
asyncio.run(with_worker(body, tts=FakeTts(), recorder=recorder))
|
|
recorder.close()
|
|
day, lines = _index(tmp_path)
|
|
said, spoke = lines
|
|
assert (said["kind"], said["who"], said["text"]) == ("said", "alice", "level 0.9")
|
|
assert (spoke["kind"], spoke["text"], spoke["cancelled"]) == ("spoke", "오늘은 맑아요. 기온은 17.5도예요.", False)
|
|
with wave.open(str(day / said["file"])) as w:
|
|
assert (w.getframerate(), w.getnchannels()) == (16000, 1)
|
|
assert 500 <= w.getnframes() * 1000 // 16000 <= 900
|
|
with wave.open(str(day / spoke["file"])) as w:
|
|
# Mono at the session's TTS rate; FakeTts makes 10 samples per character of each sentence.
|
|
assert (w.getframerate(), w.getnchannels()) == (24000, 1)
|
|
assert w.getnframes() == (len("오늘은 맑아요.") + len("기온은 17.5도예요.")) * 10
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_without_a_recorder_nothing_is_written(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws, stt={"partials": False})
|
|
await stream(ws, "alice", 0.9, 600)
|
|
await ws.send(json.dumps({"type": "speaking_end", "speaker": "alice"}))
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final")
|
|
|
|
asyncio.run(with_worker(body, tts=FakeTts()))
|
|
assert list(tmp_path.iterdir()) == []
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_recordings_older_than_the_keep_days_are_removed_at_start(tmp_path):
|
|
from datetime import date, datetime, timedelta
|
|
from voice_worker.recordings import KEEP_DAYS, Recorder
|
|
|
|
today = date(2026, 11, 16)
|
|
keep = [today - timedelta(days=i) for i in range(KEEP_DAYS)]
|
|
old = today - timedelta(days=KEEP_DAYS)
|
|
for d in keep + [old]:
|
|
(tmp_path / d.isoformat()).mkdir()
|
|
(tmp_path / "notes").mkdir() # not a day folder: left alone
|
|
Recorder(tmp_path, now=datetime(2026, 11, 16, 12).timestamp()).close()
|
|
assert sorted(p.name for p in tmp_path.iterdir()) == sorted([d.isoformat() for d in keep] + ["notes"])
|