Files
javis_bot/tests/test_voice_worker_server.py
javis-bot 420c9c73f3 fix: stopped sentences recorded only as far as heard, search words as plain query, paused video said so
- Playback sends how long the clip playing when a request was stopped had played (tts_played
  last_ms); the worker cuts the last sentence of that recording to it.
- botBrowser open takes the search words as plain `query`: the tool puts them, encoded, into the
  address's empty search parameter or types them into the site's search box. The description
  tells the model to search this way and never put words into an address itself (live: 아이유
  hand-encoded into 아이우). Eval (qwen3:8b, 3 runs): right words reach YouTube 1/3 -> 3/3.
- When a page's main video does not play (also after the tool pressed play, or it failed to load),
  a note after the page says it is not playing and not to say it plays. Eval: a paused video
  reported as playing 1/3 -> 0/3.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 16:12:04 +09:00

499 lines
21 KiB
Python

"""Voice worker end to end over a real local WebSocket, with fake STT/VAD/TTS engines.
The fake VAD reads speech probability off the amplitude and the fake transcriber names a clip by its
"level": each speaker sends a tone whose amplitude is the level (what the fake VAD reads) and whose
pitch is level x 1000 Hz (what the fake transcriber reads, since the worker evens out loudness).
"""
from __future__ import annotations
import asyncio
import json
import time
import numpy as np
import pytest
from voice_worker.protocol import decode_tts_frame, encode_audio_frame
from voice_worker.server import CLOSE_BAD_HELLO, CLOSE_BAD_TOKEN, VoiceWorker
websockets = pytest.importorskip("websockets")
from websockets.asyncio.client import connect # noqa: E402
TOKEN = "test-token"
class AmplitudeVad:
def prob(self, window):
return float(np.max(np.abs(window)))
class LevelTranscriber:
def __init__(self):
self.calls = []
def transcribe(self, clips, opts):
self.calls.append((opts, len(clips)))
return [f"level {np.argmax(np.abs(np.fft.rfft(c))) * 16000 / len(c) / 1000:.1f}" for c in clips]
class FakeTts:
"""10 samples per character; optional delay per sentence; can fail on a marker."""
def __init__(self, delay: float = 0.0):
self.delay = delay
self.texts = []
self.speed = 1.5
self.speeds = []
def synthesize(self, text, rate):
if self.delay:
time.sleep(self.delay)
if "FAIL" in text:
raise RuntimeError("engine broke")
self.texts.append(text)
self.speeds.append(self.speed)
return np.full(len(text) * 10, 1000, dtype=np.int16)
def pcm(level: float, ms: int, rate=16000, channels=1) -> bytes:
n = rate * ms // 1000
# 20 ms frames hold whole cycles at these pitches, so frames join without a click.
x = level * np.sin(2 * np.pi * level * 1000 * np.arange(n) / rate)
return (np.repeat(x, channels) * 32767).astype("<i2").tobytes()
async def with_worker(body, transcriber=None, tts=None, **kw):
worker = VoiceWorker(TOKEN, transcriber if transcriber is not None else LevelTranscriber(), AmplitudeVad,
tts, **kw)
server = await worker.serve("127.0.0.1", 0)
port = server.sockets[0].getsockname()[1]
try:
return await asyncio.wait_for(body(f"ws://127.0.0.1:{port}"), 15)
finally:
server.close()
await server.wait_closed()
await worker.stop()
async def hello(ws, **extra):
await ws.send(json.dumps({"type": "hello", "token": TOKEN, "audio": {"rate": 16000, "channels": 1},
"tts": {"rate": 24000, "channels": 2}, **extra}))
return json.loads(await ws.recv())
async def recv_until(ws, pred, limit=200):
seen = []
for _ in range(limit):
raw = await ws.recv()
msg = raw if isinstance(raw, bytes) else json.loads(raw)
seen.append(msg)
if pred(msg):
return seen
raise AssertionError("condition never met")
async def stream(ws, speaker: str, level: float, ms: int, realtime: bool = False):
for _ in range(ms // 20):
await ws.send(encode_audio_frame(speaker, pcm(level, 20)))
if realtime:
await asyncio.sleep(0.02)
@pytest.mark.unit
def test_wrong_token_is_refused():
async def body(url):
async with connect(url) as ws:
await ws.send(json.dumps({"type": "hello", "token": "nope"}))
with pytest.raises(websockets.ConnectionClosed) as e:
await ws.recv()
return e.value.rcvd.code
assert asyncio.run(with_worker(body)) == CLOSE_BAD_TOKEN
@pytest.mark.unit
def test_silent_client_is_dropped_after_the_hello_timeout():
async def body(url):
async with connect(url) as ws:
with pytest.raises(websockets.ConnectionClosed) as e:
await ws.recv()
return e.value.rcvd.code
assert asyncio.run(with_worker(body, hello_timeout_s=0.2)) == CLOSE_BAD_HELLO
@pytest.mark.unit
def test_non_loopback_addresses_are_refused():
worker = VoiceWorker(TOKEN, LevelTranscriber(), AmplitudeVad)
with pytest.raises(ValueError):
asyncio.run(worker.serve("0.0.0.0", 0))
@pytest.mark.unit
def test_hello_ok_reports_readiness_and_effective_settings():
async def body(url):
async with connect(url) as ws:
return await hello(ws, stt={"hotwords": ["자비스"], "end_silence_ms": 250})
ok = asyncio.run(with_worker(body))
assert ok["type"] == "hello_ok"
assert ok["stt_ready"] is True and ok["tts_ready"] is False
assert ok["stt"]["hotwords"] == ["자비스"] and ok["stt"]["end_silence_ms"] == 250
assert ok["tts"] == {"rate": 24000, "channels": 2}
@pytest.mark.unit
def test_two_speakers_each_get_their_own_final():
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await stream(ws, "alice", 0.9, 600)
await stream(ws, "bob", 0.6, 600)
await ws.send(json.dumps({"type": "speaking_end", "speaker": "alice"}))
await ws.send(json.dumps({"type": "speaking_end", "speaker": "bob"}))
seen = await recv_until(ws, lambda m: m.get("type") == "final" and m["speaker"] == "bob")
if not any(m.get("type") == "final" and m["speaker"] == "alice" for m in seen):
seen += await recv_until(ws, lambda m: m.get("type") == "final")
return seen
seen = asyncio.run(with_worker(body))
finals = {m["speaker"]: m for m in seen if m.get("type") == "final"}
assert finals["alice"]["text"] == "level 0.9" and finals["bob"]["text"] == "level 0.6"
assert {m["end_reason"] for m in finals.values()} == {"speaking_end"}
assert all(m["utt"] == 1 and 500 <= m["audio_ms"] <= 900 for m in finals.values())
starts = [m for m in seen if m.get("type") == "speech_start"]
assert sorted(m["speaker"] for m in starts) == ["alice", "bob"]
assert not any(m.get("type") == "partial" for m in seen)
@pytest.mark.unit
def test_audio_that_stops_without_speaking_end_still_ends():
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False, "end_silence_ms": 200})
await stream(ws, "alice", 0.9, 500)
return await recv_until(ws, lambda m: m.get("type") == "final")
final = asyncio.run(with_worker(body))[-1]
assert final["end_reason"] == "idle" and final["text"] == "level 0.9"
@pytest.mark.unit
def test_partials_flow_while_speaking_and_stop_when_turned_off():
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partial_interval_ms": 100})
await stream(ws, "alice", 0.9, 800, realtime=True)
first = await recv_until(ws, lambda m: m.get("type") == "partial")
await ws.send(json.dumps({"type": "config", "partials": False}))
# Drain whatever was already on its way before the setting changed.
while True:
try:
await asyncio.wait_for(ws.recv(), 0.3)
except asyncio.TimeoutError:
break
await stream(ws, "alice", 0.9, 600, realtime=True)
await ws.send(json.dumps({"type": "speaking_end", "speaker": "alice"}))
rest = await recv_until(ws, lambda m: m.get("type") == "final")
return first, rest
first, rest = asyncio.run(with_worker(body))
assert first[-1]["text"] == "level 0.9" and first[-1]["utt"] == 1
assert not any(m.get("type") == "partial" for m in rest)
assert rest[-1]["type"] == "final"
@pytest.mark.unit
def test_tts_streams_one_chunk_per_sentence_in_the_session_format():
tts = FakeTts()
async def body(url):
async with connect(url) as ws:
await hello(ws)
await ws.send(json.dumps({"type": "tts", "id": "r1", "text": "오늘은 맑아요. 기온은 17.5도예요."}))
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
seen = asyncio.run(with_worker(body, tts=tts))
chunks = [decode_tts_frame(m) for m in seen if isinstance(m, bytes)]
assert [(c[0], c[1]) for c in chunks] == [("r1", 0), ("r1", 1)]
assert tts.texts == ["오늘은 맑아요.", "기온은 17.5도예요."]
# Stereo: every mono sample twice, 16-bit.
assert len(chunks[0][2]) == len("오늘은 맑아요.") * 10 * 2 * 2
assert seen[-1] == {"type": "tts_done", "id": "r1", "chunks": 2, "cancelled": False, "failed": False}
@pytest.mark.unit
def test_tts_text_can_arrive_in_pieces():
tts = FakeTts()
async def body(url):
async with connect(url) as ws:
await hello(ws)
for piece, more in [("첫 번째 문장", True), ("이에요. 두 번", True), ("째 문장이에요.", False)]:
await ws.send(json.dumps({"type": "tts", "id": "s", "text": piece, "more": more}))
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
seen = asyncio.run(with_worker(body, tts=tts))
assert tts.texts == ["첫 번째 문장이에요.", "두 번째 문장이에요."]
assert seen[-1]["chunks"] == 2
@pytest.mark.unit
def test_cancel_stops_the_remaining_sentences():
tts = FakeTts(delay=0.15)
async def body(url):
async with connect(url) as ws:
await hello(ws)
text = " ".join(f"이것은 {i}번째 문장입니다." for i in range(6))
await ws.send(json.dumps({"type": "tts", "id": "c", "text": text}))
first = await recv_until(ws, lambda m: isinstance(m, bytes))
await ws.send(json.dumps({"type": "tts_cancel", "id": "c"}))
return first + await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
seen = asyncio.run(with_worker(body, tts=tts))
done = seen[-1]
assert done["cancelled"] is True and done["chunks"] < 6
assert sum(isinstance(m, bytes) for m in seen) == done["chunks"]
@pytest.mark.unit
def test_a_failing_sentence_ends_the_request_as_failed():
tts = FakeTts()
async def body(url):
async with connect(url) as ws:
await hello(ws)
await ws.send(json.dumps({"type": "tts", "id": "f", "text": "첫 문장입니다. FAIL 문장입니다. 셋째 문장입니다."}))
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
seen = asyncio.run(with_worker(body, tts=tts))
errors = [m for m in seen if isinstance(m, dict) and m.get("type") == "error"]
assert errors and errors[0]["code"] == "tts_failed" and errors[0]["id"] == "f"
assert seen[-1]["failed"] is True and seen[-1]["chunks"] == 1
assert tts.texts == ["첫 문장입니다."]
@pytest.mark.unit
def test_a_new_speech_speed_applies_to_the_sentences_made_after_it():
tts = FakeTts()
async def body(url):
async with connect(url) as ws:
await hello(ws)
await ws.send(json.dumps({"type": "tts", "id": "a", "text": "보통 속도 문장이에요."}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
await ws.send(json.dumps({"type": "tts_speed", "speed": 1.8}))
out = []
for bad in (2.5, 0.2, "fast", True):
await ws.send(json.dumps({"type": "tts_speed", "speed": bad}))
out.append(json.loads(await ws.recv()))
await ws.send(json.dumps({"type": "tts", "id": "b", "text": "빨라진 문장이에요."}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
return out
errors = asyncio.run(with_worker(body, tts=tts))
assert [e["code"] for e in errors] == ["bad_message"] * 4
assert tts.speeds == [1.5, 1.8], "a refused speed leaves the last good one"
@pytest.mark.unit
def test_tts_without_an_engine_reports_unavailable():
async def body(url):
async with connect(url) as ws:
await hello(ws)
await ws.send(json.dumps({"type": "tts", "id": "x", "text": "안녕하세요."}))
return json.loads(await ws.recv())
msg = asyncio.run(with_worker(body))
assert msg["type"] == "error" and msg["code"] == "tts_unavailable" and msg["id"] == "x"
@pytest.mark.unit
def test_bad_messages_get_an_error_and_the_connection_stays_open():
async def body(url):
async with connect(url) as ws:
await hello(ws)
out = []
for raw in ["{not json", json.dumps({"type": "dance"}), json.dumps({"type": "config", "final_beam": 99})]:
await ws.send(raw)
out.append(json.loads(await ws.recv()))
await ws.send(b"\x01\x00")
out.append(json.loads(await ws.recv()))
await ws.send(json.dumps({"type": "config", "final_beam": 3}))
await ws.send(json.dumps({"type": "dance"}))
out.append(json.loads(await ws.recv()))
return out
out = asyncio.run(with_worker(body))
assert [m["code"] for m in out] == ["bad_message", "bad_message", "bad_message", "bad_frame", "bad_message"]
@pytest.mark.unit
def test_48k_stereo_input_reaches_the_model_at_16k():
t = LevelTranscriber()
lengths = []
orig = t.transcribe
def spy(clips, opts):
lengths.extend(len(c) for c in clips)
return orig(clips, opts)
t.transcribe = spy
async def body(url):
async with connect(url) as ws:
await ws.send(json.dumps({"type": "hello", "token": TOKEN, "stt": {"partials": False}}))
await ws.recv()
for _ in range(30): # 600 ms
await ws.send(encode_audio_frame("a", pcm(0.9, 20, rate=48000, channels=2)))
await ws.send(json.dumps({"type": "speaking_end", "speaker": "a"}))
return await recv_until(ws, lambda m: m.get("type") == "final")
final = asyncio.run(with_worker(body, transcriber=t))[-1]
assert final["text"] == "level 0.9"
assert 0.5 * 16000 <= lengths[0] <= 0.9 * 16000
@pytest.mark.unit
def test_speakers_who_leave_free_their_place_and_finish_their_utterance():
from voice_worker.server import MAX_SPEAKERS
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
seen = []
for i in range(MAX_SPEAKERS + 8):
await stream(ws, f"user{i}", 0.9, 400)
# Leaving mid-utterance still delivers that utterance's final.
await ws.send(json.dumps({"type": "speaker_leave", "speaker": f"user{i}"}))
seen += await recv_until(ws, lambda m: m.get("type") in ("final", "error"))
return seen
seen = asyncio.run(with_worker(body))
assert not [m for m in seen if m.get("type") == "error"]
finals = [m for m in seen if m.get("type") == "final"]
assert [m["speaker"] for m in finals] == [f"user{i}" for i in range(len(finals))]
assert len(finals) > MAX_SPEAKERS and all(m["text"] == "level 0.9" for m in finals)
def _index(root):
import pathlib
days = [d for d in pathlib.Path(root).iterdir() if d.is_dir()]
assert len(days) == 1
lines = [json.loads(x) for x in (days[0] / "index.jsonl").read_text(encoding="utf-8").splitlines()]
return days[0], lines
@pytest.mark.unit
def test_with_recordings_on_each_utterance_and_spoken_answer_is_kept_as_a_wav_with_its_text(tmp_path):
import wave
from voice_worker.recordings import Recorder
recorder = Recorder(tmp_path)
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await ws.send(json.dumps({"type": "member_join", "user": "alice", "name": "Alice"}))
await stream(ws, "alice", 0.9, 600)
await ws.send(json.dumps({"type": "speaking_end", "speaker": "alice"}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final")
await ws.send(json.dumps({"type": "tts", "id": "r1", "text": "오늘은 맑아요. 기온은 17.5도예요."}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
# The app played both sentences to the end.
await ws.send(json.dumps({"type": "tts_played", "id": "r1", "played": 2, "cancelled": False}))
await asyncio.sleep(0.2)
asyncio.run(with_worker(body, tts=FakeTts(), recorder=recorder))
recorder.close()
day, lines = _index(tmp_path)
said, spoke = lines
assert (said["kind"], said["who"], said["text"]) == ("said", "alice", "level 0.9")
assert (spoke["kind"], spoke["text"], spoke["cancelled"]) == ("spoke", "오늘은 맑아요. 기온은 17.5도예요.", False)
with wave.open(str(day / said["file"])) as w:
assert (w.getframerate(), w.getnchannels()) == (16000, 1)
assert 500 <= w.getnframes() * 1000 // 16000 <= 900
with wave.open(str(day / spoke["file"])) as w:
# Mono at the session's TTS rate; FakeTts makes 10 samples per character of each sentence.
assert (w.getframerate(), w.getnchannels()) == (24000, 1)
assert w.getnframes() == (len("오늘은 맑아요.") + len("기온은 17.5도예요.")) * 10
@pytest.mark.unit
def test_without_a_recorder_nothing_is_written(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await stream(ws, "alice", 0.9, 600)
await ws.send(json.dumps({"type": "speaking_end", "speaker": "alice"}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final")
asyncio.run(with_worker(body, tts=FakeTts()))
assert list(tmp_path.iterdir()) == []
@pytest.mark.unit
def test_recordings_older_than_the_keep_days_are_removed_at_start(tmp_path):
from datetime import date, datetime, timedelta
from voice_worker.recordings import KEEP_DAYS, Recorder
today = date(2026, 11, 16)
keep = [today - timedelta(days=i) for i in range(KEEP_DAYS)]
old = today - timedelta(days=KEEP_DAYS)
for d in keep + [old]:
(tmp_path / d.isoformat()).mkdir()
(tmp_path / "notes").mkdir() # not a day folder: left alone
Recorder(tmp_path, now=datetime(2026, 11, 16, 12).timestamp()).close()
assert sorted(p.name for p in tmp_path.iterdir()) == sorted([d.isoformat() for d in keep] + ["notes"])
@pytest.mark.unit
def test_a_spoken_answer_is_kept_only_as_far_as_the_app_played_it(tmp_path):
import wave
from voice_worker.recordings import Recorder
recorder = Recorder(tmp_path)
async def body(url):
async with connect(url) as ws:
await hello(ws)
# Made in full, then stopped while its first sentence played.
await ws.send(json.dumps({"type": "tts", "id": "r1", "text": "오늘은 맑아요. 기온은 17.5도예요."}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
# Stopped 2 ms into its first sentence (FakeTts: 10 samples per character, 24 kHz).
await ws.send(json.dumps({"type": "tts_played", "id": "r1", "played": 1, "cancelled": True, "last_ms": 2}))
# Stopped before any of it was heard: nothing to keep.
await ws.send(json.dumps({"type": "tts", "id": "r2", "text": "두 번째 답이에요."}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
await ws.send(json.dumps({"type": "tts_played", "id": "r2", "played": 0, "cancelled": True}))
# Never reported as played (an older app): not kept as if it had been heard.
await ws.send(json.dumps({"type": "tts", "id": "r3", "text": "세 번째 답이에요."}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
await asyncio.sleep(0.2)
asyncio.run(with_worker(body, tts=FakeTts(), recorder=recorder))
recorder.close()
day, lines = _index(tmp_path)
assert [(l["id"], l["cancelled"]) for l in lines] == [("r1", True)]
with wave.open(str(day / lines[0]["file"])) as w:
assert w.getnframes() == 24000 * 2 // 1000, "only the part of the sentence that was played"
@pytest.mark.unit
def test_old_recordings_are_also_removed_while_the_worker_keeps_running(tmp_path):
from datetime import datetime, timedelta
import numpy as np
from voice_worker.recordings import KEEP_DAYS, Recorder
start = datetime(2026, 11, 16, 12)
r = Recorder(tmp_path, now=start.timestamp())
r.said(start.timestamp(), "1", "A", "vc", "첫날", np.zeros(1600))
later = start + timedelta(days=KEEP_DAYS) # the first recording of a day past the keep days
r.said(later.timestamp(), "1", "A", "vc", "나중", np.zeros(1600))
r.close()
assert sorted(p.name for p in tmp_path.iterdir()) == [later.date().isoformat()]