Files
javis_bot/tests/test_voice_worker_server.py
javis-bot cbe898d3d0 feat(app): log screen saves (shown lines, conversation, voice recordings), stop-or-finish on a new request, video pages that play
- 로그 screen: "보이는 글 txt로 저장" (logs_save_text), a 대화 view that saves the days'
  conversation with every trace in full (history_export), and a 목소리 view with the
  voice.record switch and the days' recordings as a zip with an index (voice_export).
- Voice worker: with JARVIS_RECORDINGS_DIR (voice.record, off by default) each final and each
  spoken answer is kept as a WAV with an index line; day folders older than 7 days removed.
- response.interrupt (default on): off, latest/important finish the running answer first.
- Bot browser: Chrome lets pages start video and sound; botBrowser reports each page's media
  state, waits for a player to start after open/click, and presses play on a video page a click
  led to when its player stayed paused (live: a clicked YouTube video stayed paused).
- webSearch reads `query` when `search_query` is missing (live: both searches of an answer failed).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 15:27:57 +09:00

449 lines
18 KiB
Python

"""Voice worker end to end over a real local WebSocket, with fake STT/VAD/TTS engines.
The fake VAD reads speech probability off the amplitude and the fake transcriber names a clip by its
"level": each speaker sends a tone whose amplitude is the level (what the fake VAD reads) and whose
pitch is level x 1000 Hz (what the fake transcriber reads, since the worker evens out loudness).
"""
from __future__ import annotations
import asyncio
import json
import time
import numpy as np
import pytest
from voice_worker.protocol import decode_tts_frame, encode_audio_frame
from voice_worker.server import CLOSE_BAD_HELLO, CLOSE_BAD_TOKEN, VoiceWorker
websockets = pytest.importorskip("websockets")
from websockets.asyncio.client import connect # noqa: E402
TOKEN = "test-token"
class AmplitudeVad:
def prob(self, window):
return float(np.max(np.abs(window)))
class LevelTranscriber:
def __init__(self):
self.calls = []
def transcribe(self, clips, opts):
self.calls.append((opts, len(clips)))
return [f"level {np.argmax(np.abs(np.fft.rfft(c))) * 16000 / len(c) / 1000:.1f}" for c in clips]
class FakeTts:
"""10 samples per character; optional delay per sentence; can fail on a marker."""
def __init__(self, delay: float = 0.0):
self.delay = delay
self.texts = []
self.speed = 1.5
self.speeds = []
def synthesize(self, text, rate):
if self.delay:
time.sleep(self.delay)
if "FAIL" in text:
raise RuntimeError("engine broke")
self.texts.append(text)
self.speeds.append(self.speed)
return np.full(len(text) * 10, 1000, dtype=np.int16)
def pcm(level: float, ms: int, rate=16000, channels=1) -> bytes:
n = rate * ms // 1000
# 20 ms frames hold whole cycles at these pitches, so frames join without a click.
x = level * np.sin(2 * np.pi * level * 1000 * np.arange(n) / rate)
return (np.repeat(x, channels) * 32767).astype("<i2").tobytes()
async def with_worker(body, transcriber=None, tts=None, **kw):
worker = VoiceWorker(TOKEN, transcriber if transcriber is not None else LevelTranscriber(), AmplitudeVad,
tts, **kw)
server = await worker.serve("127.0.0.1", 0)
port = server.sockets[0].getsockname()[1]
try:
return await asyncio.wait_for(body(f"ws://127.0.0.1:{port}"), 15)
finally:
server.close()
await server.wait_closed()
await worker.stop()
async def hello(ws, **extra):
await ws.send(json.dumps({"type": "hello", "token": TOKEN, "audio": {"rate": 16000, "channels": 1},
"tts": {"rate": 24000, "channels": 2}, **extra}))
return json.loads(await ws.recv())
async def recv_until(ws, pred, limit=200):
seen = []
for _ in range(limit):
raw = await ws.recv()
msg = raw if isinstance(raw, bytes) else json.loads(raw)
seen.append(msg)
if pred(msg):
return seen
raise AssertionError("condition never met")
async def stream(ws, speaker: str, level: float, ms: int, realtime: bool = False):
for _ in range(ms // 20):
await ws.send(encode_audio_frame(speaker, pcm(level, 20)))
if realtime:
await asyncio.sleep(0.02)
@pytest.mark.unit
def test_wrong_token_is_refused():
async def body(url):
async with connect(url) as ws:
await ws.send(json.dumps({"type": "hello", "token": "nope"}))
with pytest.raises(websockets.ConnectionClosed) as e:
await ws.recv()
return e.value.rcvd.code
assert asyncio.run(with_worker(body)) == CLOSE_BAD_TOKEN
@pytest.mark.unit
def test_silent_client_is_dropped_after_the_hello_timeout():
async def body(url):
async with connect(url) as ws:
with pytest.raises(websockets.ConnectionClosed) as e:
await ws.recv()
return e.value.rcvd.code
assert asyncio.run(with_worker(body, hello_timeout_s=0.2)) == CLOSE_BAD_HELLO
@pytest.mark.unit
def test_non_loopback_addresses_are_refused():
worker = VoiceWorker(TOKEN, LevelTranscriber(), AmplitudeVad)
with pytest.raises(ValueError):
asyncio.run(worker.serve("0.0.0.0", 0))
@pytest.mark.unit
def test_hello_ok_reports_readiness_and_effective_settings():
async def body(url):
async with connect(url) as ws:
return await hello(ws, stt={"hotwords": ["자비스"], "end_silence_ms": 250})
ok = asyncio.run(with_worker(body))
assert ok["type"] == "hello_ok"
assert ok["stt_ready"] is True and ok["tts_ready"] is False
assert ok["stt"]["hotwords"] == ["자비스"] and ok["stt"]["end_silence_ms"] == 250
assert ok["tts"] == {"rate": 24000, "channels": 2}
@pytest.mark.unit
def test_two_speakers_each_get_their_own_final():
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await stream(ws, "alice", 0.9, 600)
await stream(ws, "bob", 0.6, 600)
await ws.send(json.dumps({"type": "speaking_end", "speaker": "alice"}))
await ws.send(json.dumps({"type": "speaking_end", "speaker": "bob"}))
seen = await recv_until(ws, lambda m: m.get("type") == "final" and m["speaker"] == "bob")
if not any(m.get("type") == "final" and m["speaker"] == "alice" for m in seen):
seen += await recv_until(ws, lambda m: m.get("type") == "final")
return seen
seen = asyncio.run(with_worker(body))
finals = {m["speaker"]: m for m in seen if m.get("type") == "final"}
assert finals["alice"]["text"] == "level 0.9" and finals["bob"]["text"] == "level 0.6"
assert {m["end_reason"] for m in finals.values()} == {"speaking_end"}
assert all(m["utt"] == 1 and 500 <= m["audio_ms"] <= 900 for m in finals.values())
starts = [m for m in seen if m.get("type") == "speech_start"]
assert sorted(m["speaker"] for m in starts) == ["alice", "bob"]
assert not any(m.get("type") == "partial" for m in seen)
@pytest.mark.unit
def test_audio_that_stops_without_speaking_end_still_ends():
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False, "end_silence_ms": 200})
await stream(ws, "alice", 0.9, 500)
return await recv_until(ws, lambda m: m.get("type") == "final")
final = asyncio.run(with_worker(body))[-1]
assert final["end_reason"] == "idle" and final["text"] == "level 0.9"
@pytest.mark.unit
def test_partials_flow_while_speaking_and_stop_when_turned_off():
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partial_interval_ms": 100})
await stream(ws, "alice", 0.9, 800, realtime=True)
first = await recv_until(ws, lambda m: m.get("type") == "partial")
await ws.send(json.dumps({"type": "config", "partials": False}))
# Drain whatever was already on its way before the setting changed.
while True:
try:
await asyncio.wait_for(ws.recv(), 0.3)
except asyncio.TimeoutError:
break
await stream(ws, "alice", 0.9, 600, realtime=True)
await ws.send(json.dumps({"type": "speaking_end", "speaker": "alice"}))
rest = await recv_until(ws, lambda m: m.get("type") == "final")
return first, rest
first, rest = asyncio.run(with_worker(body))
assert first[-1]["text"] == "level 0.9" and first[-1]["utt"] == 1
assert not any(m.get("type") == "partial" for m in rest)
assert rest[-1]["type"] == "final"
@pytest.mark.unit
def test_tts_streams_one_chunk_per_sentence_in_the_session_format():
tts = FakeTts()
async def body(url):
async with connect(url) as ws:
await hello(ws)
await ws.send(json.dumps({"type": "tts", "id": "r1", "text": "오늘은 맑아요. 기온은 17.5도예요."}))
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
seen = asyncio.run(with_worker(body, tts=tts))
chunks = [decode_tts_frame(m) for m in seen if isinstance(m, bytes)]
assert [(c[0], c[1]) for c in chunks] == [("r1", 0), ("r1", 1)]
assert tts.texts == ["오늘은 맑아요.", "기온은 17.5도예요."]
# Stereo: every mono sample twice, 16-bit.
assert len(chunks[0][2]) == len("오늘은 맑아요.") * 10 * 2 * 2
assert seen[-1] == {"type": "tts_done", "id": "r1", "chunks": 2, "cancelled": False, "failed": False}
@pytest.mark.unit
def test_tts_text_can_arrive_in_pieces():
tts = FakeTts()
async def body(url):
async with connect(url) as ws:
await hello(ws)
for piece, more in [("첫 번째 문장", True), ("이에요. 두 번", True), ("째 문장이에요.", False)]:
await ws.send(json.dumps({"type": "tts", "id": "s", "text": piece, "more": more}))
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
seen = asyncio.run(with_worker(body, tts=tts))
assert tts.texts == ["첫 번째 문장이에요.", "두 번째 문장이에요."]
assert seen[-1]["chunks"] == 2
@pytest.mark.unit
def test_cancel_stops_the_remaining_sentences():
tts = FakeTts(delay=0.15)
async def body(url):
async with connect(url) as ws:
await hello(ws)
text = " ".join(f"이것은 {i}번째 문장입니다." for i in range(6))
await ws.send(json.dumps({"type": "tts", "id": "c", "text": text}))
first = await recv_until(ws, lambda m: isinstance(m, bytes))
await ws.send(json.dumps({"type": "tts_cancel", "id": "c"}))
return first + await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
seen = asyncio.run(with_worker(body, tts=tts))
done = seen[-1]
assert done["cancelled"] is True and done["chunks"] < 6
assert sum(isinstance(m, bytes) for m in seen) == done["chunks"]
@pytest.mark.unit
def test_a_failing_sentence_ends_the_request_as_failed():
tts = FakeTts()
async def body(url):
async with connect(url) as ws:
await hello(ws)
await ws.send(json.dumps({"type": "tts", "id": "f", "text": "첫 문장입니다. FAIL 문장입니다. 셋째 문장입니다."}))
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
seen = asyncio.run(with_worker(body, tts=tts))
errors = [m for m in seen if isinstance(m, dict) and m.get("type") == "error"]
assert errors and errors[0]["code"] == "tts_failed" and errors[0]["id"] == "f"
assert seen[-1]["failed"] is True and seen[-1]["chunks"] == 1
assert tts.texts == ["첫 문장입니다."]
@pytest.mark.unit
def test_a_new_speech_speed_applies_to_the_sentences_made_after_it():
tts = FakeTts()
async def body(url):
async with connect(url) as ws:
await hello(ws)
await ws.send(json.dumps({"type": "tts", "id": "a", "text": "보통 속도 문장이에요."}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
await ws.send(json.dumps({"type": "tts_speed", "speed": 1.8}))
out = []
for bad in (2.5, 0.2, "fast", True):
await ws.send(json.dumps({"type": "tts_speed", "speed": bad}))
out.append(json.loads(await ws.recv()))
await ws.send(json.dumps({"type": "tts", "id": "b", "text": "빨라진 문장이에요."}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
return out
errors = asyncio.run(with_worker(body, tts=tts))
assert [e["code"] for e in errors] == ["bad_message"] * 4
assert tts.speeds == [1.5, 1.8], "a refused speed leaves the last good one"
@pytest.mark.unit
def test_tts_without_an_engine_reports_unavailable():
async def body(url):
async with connect(url) as ws:
await hello(ws)
await ws.send(json.dumps({"type": "tts", "id": "x", "text": "안녕하세요."}))
return json.loads(await ws.recv())
msg = asyncio.run(with_worker(body))
assert msg["type"] == "error" and msg["code"] == "tts_unavailable" and msg["id"] == "x"
@pytest.mark.unit
def test_bad_messages_get_an_error_and_the_connection_stays_open():
async def body(url):
async with connect(url) as ws:
await hello(ws)
out = []
for raw in ["{not json", json.dumps({"type": "dance"}), json.dumps({"type": "config", "final_beam": 99})]:
await ws.send(raw)
out.append(json.loads(await ws.recv()))
await ws.send(b"\x01\x00")
out.append(json.loads(await ws.recv()))
await ws.send(json.dumps({"type": "config", "final_beam": 3}))
await ws.send(json.dumps({"type": "dance"}))
out.append(json.loads(await ws.recv()))
return out
out = asyncio.run(with_worker(body))
assert [m["code"] for m in out] == ["bad_message", "bad_message", "bad_message", "bad_frame", "bad_message"]
@pytest.mark.unit
def test_48k_stereo_input_reaches_the_model_at_16k():
t = LevelTranscriber()
lengths = []
orig = t.transcribe
def spy(clips, opts):
lengths.extend(len(c) for c in clips)
return orig(clips, opts)
t.transcribe = spy
async def body(url):
async with connect(url) as ws:
await ws.send(json.dumps({"type": "hello", "token": TOKEN, "stt": {"partials": False}}))
await ws.recv()
for _ in range(30): # 600 ms
await ws.send(encode_audio_frame("a", pcm(0.9, 20, rate=48000, channels=2)))
await ws.send(json.dumps({"type": "speaking_end", "speaker": "a"}))
return await recv_until(ws, lambda m: m.get("type") == "final")
final = asyncio.run(with_worker(body, transcriber=t))[-1]
assert final["text"] == "level 0.9"
assert 0.5 * 16000 <= lengths[0] <= 0.9 * 16000
@pytest.mark.unit
def test_speakers_who_leave_free_their_place_and_finish_their_utterance():
from voice_worker.server import MAX_SPEAKERS
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
seen = []
for i in range(MAX_SPEAKERS + 8):
await stream(ws, f"user{i}", 0.9, 400)
# Leaving mid-utterance still delivers that utterance's final.
await ws.send(json.dumps({"type": "speaker_leave", "speaker": f"user{i}"}))
seen += await recv_until(ws, lambda m: m.get("type") in ("final", "error"))
return seen
seen = asyncio.run(with_worker(body))
assert not [m for m in seen if m.get("type") == "error"]
finals = [m for m in seen if m.get("type") == "final"]
assert [m["speaker"] for m in finals] == [f"user{i}" for i in range(len(finals))]
assert len(finals) > MAX_SPEAKERS and all(m["text"] == "level 0.9" for m in finals)
def _index(root):
import pathlib
days = [d for d in pathlib.Path(root).iterdir() if d.is_dir()]
assert len(days) == 1
lines = [json.loads(x) for x in (days[0] / "index.jsonl").read_text(encoding="utf-8").splitlines()]
return days[0], lines
@pytest.mark.unit
def test_with_recordings_on_each_utterance_and_spoken_answer_is_kept_as_a_wav_with_its_text(tmp_path):
import wave
from voice_worker.recordings import Recorder
recorder = Recorder(tmp_path)
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await ws.send(json.dumps({"type": "member_join", "user": "alice", "name": "Alice"}))
await stream(ws, "alice", 0.9, 600)
await ws.send(json.dumps({"type": "speaking_end", "speaker": "alice"}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final")
await ws.send(json.dumps({"type": "tts", "id": "r1", "text": "오늘은 맑아요. 기온은 17.5도예요."}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
asyncio.run(with_worker(body, tts=FakeTts(), recorder=recorder))
recorder.close()
day, lines = _index(tmp_path)
said, spoke = lines
assert (said["kind"], said["who"], said["text"]) == ("said", "alice", "level 0.9")
assert (spoke["kind"], spoke["text"], spoke["cancelled"]) == ("spoke", "오늘은 맑아요. 기온은 17.5도예요.", False)
with wave.open(str(day / said["file"])) as w:
assert (w.getframerate(), w.getnchannels()) == (16000, 1)
assert 500 <= w.getnframes() * 1000 // 16000 <= 900
with wave.open(str(day / spoke["file"])) as w:
# Mono at the session's TTS rate; FakeTts makes 10 samples per character of each sentence.
assert (w.getframerate(), w.getnchannels()) == (24000, 1)
assert w.getnframes() == (len("오늘은 맑아요.") + len("기온은 17.5도예요.")) * 10
@pytest.mark.unit
def test_without_a_recorder_nothing_is_written(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await stream(ws, "alice", 0.9, 600)
await ws.send(json.dumps({"type": "speaking_end", "speaker": "alice"}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final")
asyncio.run(with_worker(body, tts=FakeTts()))
assert list(tmp_path.iterdir()) == []
@pytest.mark.unit
def test_recordings_older_than_the_keep_days_are_removed_at_start(tmp_path):
from datetime import date, datetime, timedelta
from voice_worker.recordings import KEEP_DAYS, Recorder
today = date(2026, 11, 16)
keep = [today - timedelta(days=i) for i in range(KEEP_DAYS)]
old = today - timedelta(days=KEEP_DAYS)
for d in keep + [old]:
(tmp_path / d.isoformat()).mkdir()
(tmp_path / "notes").mkdir() # not a day folder: left alone
Recorder(tmp_path, now=datetime(2026, 11, 16, 12).timestamp()).close()
assert sorted(p.name for p in tmp_path.iterdir()) == sorted([d.isoformat() for d in keep] + ["notes"])