Files
javis_bot/tests/test_voice_worker_brain.py
javis-bot 628d89c4f1 feat(voice_worker): finals are also decoded with the recent conversation as Whisper's prompt
The words of the topic were the ones misheard live (화면공이 for 화면 공유, 계산지 for 계산기), and
five models made the same mistakes. Each final is decoded as before and again with the last 180 s
of the channel's conversation (everyone's finals and Jarvis's speech, at most context_chars=200
characters) as the previous-text prompt; the prompted text is used only when the plain one was kept,
stays within half its letters and keeps every hotword count, as for the hint.

Measured on the game PC (RTX 5070 Ti, 100 recordings of the user, 16 with their own text): CER
12.0% -> 7.5%, empty results unchanged, noise clips with text unchanged (16/108), final decode
p95 170 -> 253 ms. Finals with a context are decoded one clip at a time (ctranslate2 needs the
start token at one position in a batch).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 19:38:10 +09:00

1847 lines
86 KiB
Python

"""Voice worker history and brain messages (voice_worker.spec.md, "History and brain").
Over a real local WebSocket with the fake STT/VAD/TTS engines of test_voice_worker_server and a
fake brain that records the history context it was given. Behaviours: presence and finals reach
the history store, chat events are kept, a reply answers with the asker's own scope, a voice answer
is stored and can be spoken, and the worker reports when it has no brain.
"""
from __future__ import annotations
import asyncio
import json
import pytest
from jarvis.memory.history import HistoryStore, Scope
from voice_worker.protocol import decode_tts_frame
from voice_worker.server import VoiceWorker
from tests.test_voice_worker_server import TOKEN, AmplitudeVad, FakeTts, LevelTranscriber, hello, recv_until, stream
websockets = pytest.importorskip("websockets")
from websockets.asyncio.client import connect # noqa: E402
class FakeBrain:
def __init__(self, answer=lambda user, text: f"answer for {text}"):
self.answer = answer
self.calls = []
def reply(self, user, text, history, chat=None, schedule=None, task=False, notes=None, notes_backend=None,
screen=None, pc=None, trace=None, cancelled=None):
self.calls.append((user, text, history))
self.cancelled = cancelled
if trace is not None:
trace.append({"kind": "tools", "names": ["getWeather"]})
self.chat, self.schedule, self.task = chat, schedule, task
self.notes, self.notes_backend, self.screen, self.pc = notes, notes_backend, screen, pc
return self.answer(user, text)
def addressed(self, user, text, names, people=None, callers=None):
self.calls.append(("addressed", user, text, names, people, callers))
return "jarvis" in text
def rank(self, candidates):
self.calls.append(("rank", [c["id"] for c in candidates]))
return [c["id"] for c in reversed(candidates)]
@pytest.fixture
def store(tmp_path):
s = HistoryStore(str(tmp_path / "history.sqlite"))
yield s
s.close()
async def with_worker(body, store=None, brain=None, tts=None, **kw):
worker = VoiceWorker(TOKEN, LevelTranscriber(), AmplitudeVad, tts, history=store, brain=brain, **kw)
server = await worker.serve("127.0.0.1", 0)
port = server.sockets[0].getsockname()[1]
try:
return await asyncio.wait_for(body(f"ws://127.0.0.1:{port}"), 15)
finally:
server.close()
await server.wait_closed()
await worker.stop()
async def send(ws, **msg):
await ws.send(json.dumps(msg))
async def say(ws, speaker, level):
await stream(ws, speaker, level, 600)
await send(ws, type="speaking_end", speaker=speaker)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final" and m["speaker"] == speaker)
def texts(records):
return [r.text for r in records]
@pytest.mark.unit
def test_finals_in_a_voice_channel_are_stored_under_the_speakers_name(store):
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await say(ws, "100", 0.9) # before Jarvis is in a channel: not stored
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="member_join", user="100", name="Alice")
await say(ws, "100", 0.6)
asyncio.run(with_worker(body, store=store))
recs = store.records_of("100")
assert texts(recs) == ["level 0.6"]
assert recs[0].author_name == "Alice" and recs[0].channel_id == "vc1" and recs[0].kind == "voice"
@pytest.mark.unit
def test_reply_answers_with_the_askers_own_scope_and_stores_the_voice_answer(store):
brain = FakeBrain()
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="member_join", user="100", name="Alice")
await say(ws, "100", 0.9)
await send(ws, type="member_join", user="200", name="Bob")
await say(ws, "200", 0.6)
await send(ws, type="reply", id="r1", user="100", text="what did we say", via="voice", speak=False)
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
await send(ws, type="reply", id="r2", user="200", text="and me", via="voice", speak=False)
seen += await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
return seen
seen = asyncio.run(with_worker(body, store=store, brain=brain))
answers = [m for m in seen if isinstance(m, dict) and m.get("type") == "reply_text"]
assert answers[0] == {"type": "reply_text", "id": "r1", "text": "answer for what did we say", "failed": False}
# Alice was there when Bob spoke, so she sees both; Bob joined after Alice spoke.
alice_ctx, bob_ctx = brain.calls[0][2], brain.calls[1][2]
assert any("Alice: level 0.9" in x for x in alice_ctx.recent)
assert any("Bob: level 0.6" in x for x in alice_ctx.recent)
assert not any("Alice: level 0.9" in x for x in bob_ctx.recent)
# The voice answer is Alice's own record now.
assert "answer for what did we say" in texts(store.records_of("100"))
@pytest.mark.unit
def test_spoken_reply_comes_back_as_tts_under_the_same_id(store):
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="reply", id="r9", user="100", text="hello", via="voice", speak=True)
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain(), tts=FakeTts()))
frames = [decode_tts_frame(m) for m in seen if isinstance(m, bytes)]
assert frames and all(rid == "r9" for rid, _, _ in frames)
done = [m for m in seen if isinstance(m, dict) and m.get("type") == "tts_done"][0]
assert done["id"] == "r9" and not done["failed"]
@pytest.mark.unit
def test_chat_answers_are_not_stored_by_the_worker_but_reported_messages_are(store):
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="c1", user="100", text="hi", via="chat", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
await send(ws, type="chat_message", channel="ch1", message_id="m1", author="100", name="Alice",
text="first", ts=1_800_000_000.0)
await send(ws, type="chat_message", channel="ch1", message_id="m2", author="900", name="Jarvis",
text="answer for hi", ts=1_800_000_001.0, to="100")
await send(ws, type="chat_edit", channel="ch1", message_id="m1", text="first, edited")
await send(ws, type="chat_message", channel="ch1", message_id="m3", author="100", name="Alice",
text="gone soon", ts=1_800_000_002.0)
await send(ws, type="chat_delete", channel="ch1", message_id="m3")
# A round trip makes sure the worker handled everything above.
await send(ws, type="reply", id="c2", user="100", text="x", via="chat", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "c2")
asyncio.run(with_worker(body, store=store, brain=FakeBrain()))
assert texts(store.records_of("100")) == ["first, edited", "answer for hi"]
@pytest.mark.unit
def test_leaving_the_voice_channel_stops_storing_and_closes_presence(store):
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="member_join", user="100", name="Alice")
await send(ws, type="voice_leave")
await say(ws, "200", 0.6)
asyncio.run(with_worker(body, store=store))
assert store.records_of("200") == []
assert store.recent("100", now=4_000_000_000, scope=Scope(), minutes=10 ** 9) == []
@pytest.mark.unit
def test_memory_off_gives_the_brain_no_history(store):
brain = FakeBrain()
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False}, memory={"off": True})
await send(ws, type="voice_join", channel="vc1")
await say(ws, "100", 0.9)
await send(ws, type="reply", id="r1", user="100", text="q", via="voice", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
await send(ws, type="memory_config", off=False)
await send(ws, type="reply", id="r2", user="100", text="q", via="voice", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r2")
asyncio.run(with_worker(body, store=store, brain=brain))
off_ctx, on_ctx = brain.calls[0][2], brain.calls[1][2]
assert off_ctx.recent == [] and off_ctx.matches == []
assert any("level 0.9" in x for x in on_ctx.recent)
@pytest.mark.unit
@pytest.mark.parametrize("answer", [lambda u, t: None, lambda u, t: 1 / 0])
def test_a_failed_brain_answer_is_reported(store, answer):
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="r1", user="100", text="q", via="voice", speak=True)
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain(answer), tts=FakeTts()))
assert seen[-1] == {"type": "reply_text", "id": "r1", "text": "", "failed": True}
@pytest.mark.unit
def test_each_answer_keeps_how_it_was_made_with_the_records_it_was_given(store):
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="member_join", user="100", name="Alice")
await say(ws, "100", 0.9)
await send(ws, type="reply", id="r1", user="100", text="what did I say", via="voice", speak=False,
notes=["반말로 해"])
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r1")
await send(ws, type="reply", id="c1", user="100", text="hi", via="chat", speak=False, channel="ch1")
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "c1")
await send(ws, type="reply", id="t1", user="100", text="task", via="task")
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "t1")
brain = FakeBrain(lambda u, t: None if t == "hi" else f"answer for {t}")
asyncio.run(with_worker(body, store=store, brain=brain))
voice, chat, task = store.traces_of("100")
assert (voice.via, voice.channel_id, voice.question, voice.answer) == \
("voice", "vc1", "what did I say", "answer for what did I say")
assert voice.ms >= 0
records = voice.steps[0]
assert records["kind"] == "records" and records["notes"] == 1
assert any("Alice: level 0.9" in x for x in records["recent"])
assert voice.steps[1:] == [{"kind": "tools", "names": ["getWeather"]}]
# A failed answer is kept too, with no answer; a task has no channel.
assert (chat.via, chat.channel_id, chat.answer) == ("chat", "ch1", "")
assert (task.via, task.channel_id, task.answer) == ("task", None, "answer for task")
@pytest.mark.unit
def test_a_trace_is_stored_from_the_request_on_and_fills_in_while_the_answer_is_made(store):
import time
during = []
def answer(user, text):
during.append((store.traces_of(user), time.time())) # the engine has added its steps
return "answer"
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="reply", id="r1", user="100", text="q", via="voice", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r1")
asyncio.run(with_worker(body, store=store, brain=FakeBrain(answer)))
[(traces, answering_at)] = during
[t] = traces
assert (t.question, t.answer, t.ms) == ("q", "", -1) # -1: still being made
assert [s["kind"] for s in t.steps] == ["records", "tools"]
assert t.ts <= answering_at
[t] = store.traces_of("100")
assert (t.answer, t.ms >= 0, t.ts) == ("answer", True, traces[0].ts)
@pytest.mark.unit
def test_without_a_brain_reply_is_an_error():
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="reply", id="r1", user="100", text="q", via="voice", speak=False)
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "error")
seen = asyncio.run(with_worker(body))
assert seen[-1]["code"] == "brain_unavailable" and seen[-1]["id"] == "r1"
@pytest.mark.unit
def test_malformed_history_messages_are_rejected(store):
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="member_join", user=5)
await send(ws, type="reply", id="r1", user="100", via="radio")
return [await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "error")
for _ in range(2)]
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain()))
assert [s[-1]["code"] for s in seen] == ["bad_message", "bad_message"]
class SlowTranscriber(LevelTranscriber):
def __init__(self, delay):
super().__init__()
self.delay = delay
def transcribe(self, clips, opts, contexts=None):
import time as _t
_t.sleep(self.delay)
return super().transcribe(clips, opts)
@pytest.mark.unit
def test_a_slow_final_keeps_the_channel_time_and_name_of_when_it_was_said(store):
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="member_join", user="100", name="Alice")
await stream(ws, "100", 0.9, 600)
await send(ws, type="speaking_end", speaker="100")
# While the model is still decoding: a late joiner arrives, then Jarvis moves.
await asyncio.sleep(0.15)
await send(ws, type="member_join", user="300", name="Late")
await send(ws, type="member_join", user="100", name="Renamed")
await send(ws, type="voice_join", channel="vc2")
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final")
worker = VoiceWorker(TOKEN, SlowTranscriber(0.6), AmplitudeVad, None, history=store)
async def run():
server = await worker.serve("127.0.0.1", 0)
port = server.sockets[0].getsockname()[1]
try:
await asyncio.wait_for(body(f"ws://127.0.0.1:{port}"), 15)
finally:
server.close()
await server.wait_closed()
await worker.stop()
asyncio.run(run())
recs = store.records_of("100")
assert [(r.channel_id, r.author_name, r.text) for r in recs] == [("vc1", "Alice", "level 0.9")]
# The late joiner was not there when it was said.
assert store.recent("300", now=4_000_000_000, scope=Scope(), minutes=10 ** 9) == []
@pytest.mark.unit
def test_presence_left_open_by_a_previous_run_is_closed_at_start(store):
store.presence_join("old", "vc1", 1_000.0) # the worker was killed with "old" still present
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="member_join", user="100", name="Alice")
await say(ws, "100", 0.9)
asyncio.run(with_worker(body, store=store))
assert texts(store.recent("100", now=4_000_000_000, scope=Scope(), minutes=10 ** 9)) == ["level 0.9"]
assert store.recent("old", now=4_000_000_000, scope=Scope(), minutes=10 ** 9) == []
@pytest.mark.unit
def test_traces_older_than_thirty_days_are_removed_at_start(store):
import time
now = time.time()
store.finish_trace(store.start_trace("voice", "vc1", "100", "old", ts=now - 31 * 86400), "a", [], ms=1)
store.finish_trace(store.start_trace("voice", "vc1", "100", "new", ts=now - 29 * 86400), "a", [], ms=1)
store.start_trace("voice", "vc1", "100", "cut off", ts=now - 60) # a killed run never finished it
async def body(url):
async with connect(url) as ws:
await hello(ws)
asyncio.run(with_worker(body, store=store))
assert [(t.question, t.ms >= 0) for t in store.traces_of("100")] == [("new", True), ("cut off", True)]
@pytest.mark.unit
def test_the_brain_prints_no_text_from_its_thread(tmp_path, monkeypatch, capfd):
pytest.importorskip("mcp") # the real brain needs the jarvis dependencies (--brain)
import sys
import threading
from jarvis.memory.history import HistoryContext
from voice_worker import brain as brain_mod
cfg = tmp_path / "config.json"
cfg.write_text(json.dumps({"db_path": str(tmp_path / "jarvis.db")}))
monkeypatch.setenv("JARVIS_CONFIG_PATH", str(cfg))
def fake_engine(db, cfg, tts, text, mem, language=None, history=None, legacy_memory=True, **kw):
print(f"answer text {text}")
print(f"debug {text}", file=sys.stderr)
# Tools print from helper threads too (the web fetch pool).
helper = threading.Thread(target=lambda: print(f"helper {text}", file=sys.stderr))
helper.start()
helper.join()
return f"answer to {text}"
import jarvis.reply.engine as engine
monkeypatch.setattr(engine, "run_reply_engine", fake_engine)
b = brain_mod.JarvisBrain()
out = []
t = threading.Thread(target=lambda: out.append(b.reply("100", "secret question", HistoryContext())))
t.start()
t.join()
print("other output stays")
captured = capfd.readouterr()
assert out == ["answer to secret question"]
assert "secret question" not in captured.out + captured.err
assert "other output stays" in captured.out
@pytest.mark.unit
def test_the_brain_turns_off_the_single_user_memory(tmp_path, monkeypatch):
pytest.importorskip("mcp") # the real brain needs the jarvis dependencies (--brain)
from jarvis.memory.history import HistoryContext
from voice_worker import brain as brain_mod
cfg = tmp_path / "config.json"
cfg.write_text(json.dumps({"db_path": str(tmp_path / "jarvis.db")}))
monkeypatch.setenv("JARVIS_CONFIG_PATH", str(cfg))
seen = {}
def fake_engine(db, cfg, tts, text, mem, language=None, history=None, legacy_memory=True, **kw):
seen["legacy_memory"] = legacy_memory
return "ok"
import jarvis.reply.engine as engine
monkeypatch.setattr(engine, "run_reply_engine", fake_engine)
brain_mod.JarvisBrain().reply("100", "q", HistoryContext())
assert seen["legacy_memory"] is False
@pytest.mark.unit
def test_the_brain_answers_in_korean_by_default_even_with_a_v1_english_voice(tmp_path, monkeypatch):
"""v1's ``tts_engine`` (an English-only Piper voice by default) must not lock v2 replies to
English: v2 speaks through the worker's own TTS. Replies come in the default reply language
(Korean) unless the asker asks for another, and tools get its ISO code."""
pytest.importorskip("mcp") # the real brain needs the jarvis dependencies (--brain)
from jarvis.memory.history import HistoryContext
from jarvis.system_prompt import default_language_directive, reply_language_directive
from voice_worker import brain as brain_mod
cfg = tmp_path / "config.json"
cfg.write_text(json.dumps({"db_path": str(tmp_path / "jarvis.db"), "tts_engine": "piper"}))
monkeypatch.setenv("JARVIS_CONFIG_PATH", str(cfg))
monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False)
seen = {}
def fake_engine(db, cfg, tts, text, mem, language=None, history=None, legacy_memory=True,
default_language=None, compact_prompt=False, **kw):
seen["directive"] = reply_language_directive(None, cfg.tts_engine, default_language)
seen["language"] = language
seen["compact"] = compact_prompt
seen["stream_browser"] = cfg.stream_browser
return "ok"
import jarvis.reply.engine as engine
monkeypatch.setattr(engine, "run_reply_engine", fake_engine)
brain_mod.JarvisBrain().reply("100", "what time is it?", HistoryContext())
assert seen["directive"] == default_language_directive("Korean")
assert seen["language"] == "ko"
# The short voice prompt, and no v1 screen-share browser (it needs v1's Node bot).
assert seen["compact"] is True and seen["stream_browser"] is False
# Without a default the brain answers in the asker's language.
brain_mod.JarvisBrain(language=None, reply_language=None).reply("100", "q", HistoryContext())
assert seen["directive"] is None and seen["language"] is None
@pytest.mark.unit
def test_the_brain_tells_tools_who_is_asking(tmp_path, monkeypatch):
pytest.importorskip("mcp") # the real brain needs the jarvis dependencies (--brain)
from jarvis.memory.history import HistoryContext
from jarvis.reply import turn_state
from voice_worker import brain as brain_mod
cfg = tmp_path / "config.json"
cfg.write_text(json.dumps({"db_path": str(tmp_path / "jarvis.db")}))
monkeypatch.setenv("JARVIS_CONFIG_PATH", str(cfg))
seen = []
def fake_engine(db, cfg, tts, text, mem, language=None, history=None, legacy_memory=True, **kw):
seen.append(turn_state.get_user())
return "ok"
import jarvis.reply.engine as engine
monkeypatch.setattr(engine, "run_reply_engine", fake_engine)
b = brain_mod.JarvisBrain()
b.reply("100", "q", HistoryContext())
b.reply("200", "q", HistoryContext())
assert seen == ["100", "200"]
assert turn_state.get_user() is None # nothing left behind on this thread
CHANNELS = {"type": "chat_channels", "bot": "900", "channels": [
{"id": "c1", "name": "일정", "description": "약속", "read": True, "send": True, "edit": True,
"react": True, "delete": False, "pin": False}]}
@pytest.mark.unit
def test_the_brain_gets_the_registered_channels_and_their_messages(store):
seen = {}
def answer(user, text):
chat = brain.chat
seen["channels"] = chat.channels()
seen["read"] = chat.read("c1", 5)
seen["search"] = chat.search(["c1"], "회의", 5)
seen["turn"] = chat.turn
return "ok"
brain = FakeBrain(answer)
async def body(url):
async with connect(url) as ws:
await hello(ws)
await ws.send(json.dumps(CHANNELS))
await send(ws, type="chat_message", channel="c1", message_id="m1", author="200", name="Bob",
text="회의는 3시", ts=1_800_000_000.0)
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
await send(ws, type="reply_delivered", id="r1", ok=True)
await send(ws, type="reply", id="r2", user="100", text="q", via="chat", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r2")
asyncio.run(with_worker(body, store=store, brain=brain))
assert [c["name"] for c in seen["channels"]] == ["일정"] and seen["channels"][0]["send"] is True
assert [(m["message_id"], m["author_name"], m["text"]) for m in seen["read"]] == [("m1", "Bob", "회의는 3시")]
assert [m["text"] for m in seen["search"]] == ["회의는 3시"]
assert seen["turn"] == 2 # the same person's second reply
@pytest.mark.unit
def test_a_chat_action_goes_to_the_app_and_its_answer_comes_back(store):
results = []
brain = FakeBrain(lambda u, t: results.append(brain.chat.act("send", "c1", text="안녕")) or "done")
async def body(url):
async with connect(url) as ws:
await hello(ws)
await ws.send(json.dumps(CHANNELS))
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "chat_action")
action = seen[-1]
await send(ws, type="chat_action_result", id=action["id"], ok=True, message_ids=["555"])
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
return action
action = asyncio.run(with_worker(body, store=store, brain=brain))
# The app records who asked for it (its activity log).
assert {k: action[k] for k in ("action", "channel", "text", "user")} == {
"action": "send", "channel": "c1", "text": "안녕", "user": "100"}
assert results == [{"ok": True, "message_ids": ["555"]}]
@pytest.mark.unit
def test_a_chat_action_the_app_never_answers_fails(store):
results = []
brain = FakeBrain(lambda u, t: results.append(brain.chat.act("send", "c1", text="x")) or "done")
async def body(url):
async with connect(url) as ws:
await hello(ws)
await ws.send(json.dumps(CHANNELS))
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
worker = VoiceWorker(TOKEN, LevelTranscriber(), AmplitudeVad, None, history=store, brain=brain,
chat_timeout_s=0.3)
async def run():
server = await worker.serve("127.0.0.1", 0)
port = server.sockets[0].getsockname()[1]
try:
await asyncio.wait_for(body(f"ws://127.0.0.1:{port}"), 15)
finally:
server.close()
await server.wait_closed()
await worker.stop()
asyncio.run(run())
assert results and results[0]["ok"] is False and results[0]["error"]
@pytest.mark.unit
def test_a_late_answer_from_an_earlier_connection_is_not_taken_for_a_new_action(store):
results = []
brain = FakeBrain(lambda u, t: results.append(brain.chat.act("send", "c1", text="x")) or "done")
worker = VoiceWorker(TOKEN, LevelTranscriber(), AmplitudeVad, None, history=store, brain=brain,
chat_timeout_s=0.5)
async def ask(url):
async with connect(url) as ws:
await hello(ws)
await ws.send(json.dumps(CHANNELS))
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "chat_action")
return ws, seen[-1]["id"]
async def run():
server = await worker.serve("127.0.0.1", 0)
url = f"ws://127.0.0.1:{server.sockets[0].getsockname()[1]}"
try:
async with connect(url) as ws: # first connection: asked, never answered, then gone
await hello(ws)
await ws.send(json.dumps(CHANNELS))
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
old = (await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "chat_action"))[-1]["id"]
async with connect(url) as ws: # second connection: the old answer arrives late
await hello(ws)
await ws.send(json.dumps(CHANNELS))
await send(ws, type="reply", id="r2", user="100", text="q", via="chat", speak=False)
new = (await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "chat_action"))[-1]["id"]
await send(ws, type="chat_action_result", id=old, ok=True, message_ids=["stale"])
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r2")
return old, new
finally:
server.close()
await server.wait_closed()
await worker.stop()
old, new = asyncio.run(asyncio.wait_for(run(), 15))
assert old != new
assert all(r.get("message_ids") != ["stale"] for r in results), results
assert results[-1]["ok"] is False
@pytest.mark.unit
def test_addressed_and_rank_answer_from_the_brain(store):
brain = FakeBrain()
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="addressed", id="q1", user="100", text="jarvis, lights off")
await send(ws, type="addressed", id="q2", user="100", text="did you see that", people=3, callers=1)
await send(ws, type="rank", id="k1", candidates=[{"id": "7", "user": "100", "text": "a"},
{"id": "8", "user": "200", "text": "b"}])
got = {}
while len(got) < 3:
m = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") in
("addressed_result", "rank_result"))
got[m[-1]["id"]] = m[-1]
return got
got = asyncio.run(with_worker(body, store=store, brain=brain))
assert got["q1"] == {"type": "addressed_result", "id": "q1", "addressed": True, "failed": False}
assert got["q2"]["addressed"] is False and got["q2"]["failed"] is False
assert got["k1"] == {"type": "rank_result", "id": "k1", "order": ["8", "7"], "failed": False}
# Who is in the channel reaches the judgement; an older app that does not say gets None.
rooms = {c[2]: c[4:] for c in brain.calls if c[0] == "addressed"}
assert rooms == {"jarvis, lights off": (None, None), "did you see that": (3, 1)}
@pytest.mark.unit
def test_addressed_and_rank_do_not_wait_behind_a_running_reply(store):
import threading
release = threading.Event()
def slow(user, text):
release.wait(10)
return "late"
brain = FakeBrain(slow)
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="r1", user="100", text="q", via="voice", speak=False)
await send(ws, type="rank", id="k1", candidates=[{"id": "7", "user": "100", "text": "a"},
{"id": "8", "user": "200", "text": "b"}])
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "rank_result")
release.set()
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
return seen
seen = asyncio.run(with_worker(body, store=store, brain=brain))
assert not any(isinstance(m, dict) and m.get("type") == "reply_text" for m in seen)
@pytest.mark.unit
@pytest.mark.parametrize("fails", [lambda *a: None, lambda *a: 1 / 0])
def test_a_failed_judgement_or_ranking_is_reported(store, fails):
brain = FakeBrain()
brain.addressed = fails
brain.rank = fails
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="addressed", id="q1", user="100", text="x")
a = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "addressed_result")
await send(ws, type="rank", id="k1", candidates=[{"id": "7", "user": "100", "text": "a"}])
r = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "rank_result")
return a[-1], r[-1]
a, r = asyncio.run(with_worker(body, store=store, brain=brain))
assert a == {"type": "addressed_result", "id": "q1", "addressed": False, "failed": True}
assert r == {"type": "rank_result", "id": "k1", "order": [], "failed": True}
@pytest.mark.unit
def test_a_reply_cancelled_while_the_brain_thinks_is_not_spoken(store):
import threading
release = threading.Event()
def slow(user, text):
release.wait(10)
return f"answer for {text}"
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="reply", id="r1", user="100", text="q", via="voice", speak=True)
await send(ws, type="tts_cancel", id="r1")
await send(ws, type="reply", id="r2", user="100", text="next", via="voice", speak=True)
await asyncio.sleep(0.1)
release.set()
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done"
and m["id"] == "r2")
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain(slow), tts=FakeTts()))
frames = {decode_tts_frame(m)[0] for m in seen if isinstance(m, bytes)}
done = {m["id"]: m for m in seen if isinstance(m, dict) and m.get("type") == "tts_done"}
assert frames == {"r2"}
assert done["r1"]["cancelled"] and not done["r2"]["cancelled"]
replies = [m["id"] for m in seen if isinstance(m, dict) and m.get("type") == "reply_text"]
assert replies == ["r1", "r2"], "the text still comes back; only the speech is skipped"
@pytest.mark.unit
def test_a_cancelled_reply_stops_the_brain_and_one_still_waiting_never_reaches_it(store):
"""A request taken back (the asker went on talking) must not hold up the next one: the brain is
told to stop the running answer, and a reply still waiting for it is never started."""
import time
started = []
def answer(user, text):
started.append(text)
if text == "first":
t0 = time.monotonic()
while not brain.cancelled() and time.monotonic() - t0 < 10:
time.sleep(0.01)
return None if brain.cancelled() else "late answer"
return f"answer for {text}"
brain = FakeBrain(answer)
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="reply", id="r1", user="100", text="first", via="voice", speak=True)
await asyncio.sleep(0.1)
await send(ws, type="reply", id="r2", user="100", text="second", via="voice", speak=True)
await send(ws, type="tts_cancel", id="r1")
await send(ws, type="tts_cancel", id="r2")
await send(ws, type="reply", id="r3", user="100", text="first second", via="voice", speak=True)
t0 = time.monotonic()
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done"
and m["id"] == "r3")
return seen, time.monotonic() - t0
seen, waited = asyncio.run(with_worker(body, store=store, brain=brain, tts=FakeTts()))
assert waited < 3, "the joined request did not wait for the cancelled answer to run out"
assert started == ["first", "first second"]
done = {m["id"]: m for m in seen if isinstance(m, dict) and m.get("type") == "tts_done"}
assert done["r1"]["cancelled"] and done["r2"]["cancelled"] and not done["r3"]["cancelled"]
frames = {decode_tts_frame(m)[0] for m in seen if isinstance(m, bytes)}
assert frames == {"r3"}
@pytest.mark.unit
def test_a_request_taken_back_does_not_use_up_the_turn_a_confirmation_needs(store):
"""Jarvis asked something; the asker says "yes" and goes on ("yes, quickly"). The app takes the
"yes" back and sends the joined text: it must still be the very next turn, or a confirmation
(pcControl, chat delete) would lapse."""
import time
turns = {}
def answer(user, text):
turns[text] = brain.pc.turn
if text == "yes":
t0 = time.monotonic()
while not brain.cancelled() and time.monotonic() - t0 < 10:
time.sleep(0.01)
return None
return f"answer for {text}"
brain = FakeBrain(answer)
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="reply", id="r1", user="100", text="open the calculator", via="voice", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
await send(ws, type="reply", id="r2", user="100", text="yes", via="voice", speak=True)
await asyncio.sleep(0.1)
await send(ws, type="tts_cancel", id="r2")
await send(ws, type="reply", id="r3", user="100", text="yes quickly", via="voice", speak=True)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done" and m["id"] == "r3")
asyncio.run(with_worker(body, store=store, brain=brain, tts=FakeTts()))
assert turns["yes quickly"] == turns["open the calculator"] + 1
@pytest.mark.unit
def test_the_brains_judgements_use_its_model_and_map_back_to_ids(monkeypatch):
import voice_worker.judges as judges
from voice_worker.brain import JarvisBrain
asked = []
def fake_llm(base_url, model, system, user, **kw):
asked.append((base_url, model, system, user))
return "2, 1" if "<requests>" in user else "Yes."
monkeypatch.setattr(judges, "call_llm_direct", fake_llm)
brain = JarvisBrain.__new__(JarvisBrain)
brain.cfg = type("Cfg", (), {"ollama_base_url": "http://s/v1", "ollama_chat_model": "m"})()
assert brain.addressed("100", "불 꺼 줘", ["자비스"]) is True
assert brain.rank([{"id": "7", "user": "1", "text": "a"}, {"id": "8", "user": "2", "text": "b"}]) == ["8", "7"]
assert asked[0][:2] == ("http://s/v1", "m") and "자비스" in asked[0][2]
assert asked[0][3] == "<utterance>불 꺼 줘</utterance>", "the heard text is fenced as data"
@pytest.mark.unit
@pytest.mark.parametrize("raw,expected", [("no", False), ("NO.", False), ("maybe", None), (None, None)])
def test_an_unclear_judgement_is_none(monkeypatch, raw, expected):
import voice_worker.judges as judges
monkeypatch.setattr(judges, "call_llm_direct", lambda *a, **k: raw)
assert judges.is_addressed("u", "m", [], "x") is expected
@pytest.mark.unit
def test_a_ranking_keeps_left_out_requests_in_order_and_fails_without_numbers(monkeypatch):
import voice_worker.judges as judges
monkeypatch.setattr(judges, "call_llm_direct", lambda *a, **k: "3 then 9")
assert judges.rank_requests("u", "m", ["a", "b", "c"]) == [2, 0, 1]
monkeypatch.setattr(judges, "call_llm_direct", lambda *a, **k: "none of them")
assert judges.rank_requests("u", "m", ["a", "b"]) is None
@pytest.mark.unit
def test_a_voice_answer_does_not_wait_behind_a_chat_answer(store):
import threading
release = threading.Event()
running = []
def answer(user, text):
running.append(text)
if text == "chat question":
release.wait(10)
return f"answer for {text}"
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="c1", user="100", text="chat question", via="chat", speak=False)
await asyncio.sleep(0.1)
await send(ws, type="reply", id="v1", user="200", text="voice question", via="voice", speak=False)
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "v1")
release.set()
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "c1")
return seen
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain(answer)))
assert seen[-1]["text"] == "answer for voice question"
assert not any(isinstance(m, dict) and m.get("id") == "c1" for m in seen), "the chat answer was still running"
@pytest.mark.unit
@pytest.mark.parametrize("via", ["voice", "chat"])
def test_answers_of_one_kind_are_written_one_at_a_time(store, via):
import threading
active, peak = [0], [0]
lock = threading.Lock()
def answer(user, text):
with lock:
active[0] += 1
peak[0] = max(peak[0], active[0])
import time as _t
_t.sleep(0.05)
with lock:
active[0] -= 1
return "ok"
async def body(url):
async with connect(url) as ws:
await hello(ws)
for i in range(3):
await send(ws, type="reply", id=f"r{i}", user="100", text="q", via=via, speak=False)
for i in range(3):
await recv_until(ws, lambda m, i=i: isinstance(m, dict) and m.get("id") == f"r{i}")
asyncio.run(with_worker(body, store=store, brain=FakeBrain(answer)))
assert peak[0] == 1
@pytest.mark.unit
def test_one_persons_voice_and_chat_answers_are_written_one_at_a_time(store):
# Their dialogue memory and delete confirmations ("the very next message") follow the order
# they asked, so a person's second question starts only after their first is answered.
import threading
release = threading.Event()
running = []
def answer(user, text):
running.append(text)
if text == "chat question":
release.wait(10)
return f"answer for {text}"
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="c1", user="100", text="chat question", via="chat", speak=False)
await asyncio.sleep(0.1)
await send(ws, type="reply", id="v1", user="100", text="voice question", via="voice", speak=False)
await asyncio.sleep(0.3)
started = list(running)
release.set()
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "v1")
return started, [m["id"] for m in seen if isinstance(m, dict) and m.get("type") == "reply_text"]
started, order = asyncio.run(with_worker(body, store=store, brain=FakeBrain(answer)))
assert started == ["chat question"], "the voice question started before the same person's chat answer"
assert order == ["c1", "v1"]
DELETABLE = {"type": "chat_channels", "bot": "900", "channels": [
{"id": "c1", "name": "일정", "description": "약속", "read": True, "send": True, "edit": True,
"react": True, "delete": True, "pin": False}]}
def delete_tool_brain(release, question="지울까요?"):
"""A brain whose reply runs the real chatChannels tool: "지워" asks to delete message m1 (and waits
for ``release`` before answering), "응, 지워" confirms. The consent check always agrees."""
from unittest.mock import patch
from jarvis.reply import turn_state
from jarvis.tools.builtin.chat_channels import ChatChannelsTool
class Cfg:
ollama_base_url = "http://x"
ollama_chat_model = "m"
results = []
def answer(user, text):
turn_state.reset()
turn_state.set_chat(brain.chat)
args = {"action": "delete", "channel": "c1", "message_id": "m1", "confirm": text == "응, 지워"}
try:
with patch("jarvis.tools.builtin.chat_channels.call_llm_direct", return_value="YES"):
res = ChatChannelsTool().execute(db=None, cfg=Cfg(), tool_args=args, system_prompt="",
original_prompt=text, redacted_text=text, max_retries=0,
user_print=lambda m: None)
finally:
turn_state.reset()
results.append((text, res.success))
if text == "지워":
release.wait(10)
return question if text == "지워" else "done"
brain = FakeBrain(answer)
return brain, results
@pytest.mark.unit
def test_a_yes_sent_before_the_delete_question_was_answered_does_not_delete(store):
import threading
release = threading.Event()
brain, results = delete_tool_brain(release)
async def body(url):
async with connect(url) as ws:
await hello(ws)
await ws.send(json.dumps(DELETABLE))
await send(ws, type="reply", id="r1", user="100", text="지워", via="chat", speak=False)
await asyncio.sleep(0.2)
# Typed before the question reached them: it cannot be their answer to it.
await send(ws, type="reply", id="r2", user="100", text="응, 지워", via="voice", speak=False)
await asyncio.sleep(0.1)
release.set()
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r2")
seen = asyncio.run(with_worker(body, store=store, brain=brain))
assert not any(isinstance(m, dict) and m.get("type") == "chat_action" for m in seen)
assert results == [("지워", True), ("응, 지워", False)]
@pytest.mark.unit
def test_a_yes_after_the_delete_question_deletes(store):
import threading
release = threading.Event()
release.set()
brain, results = delete_tool_brain(release)
async def body(url):
async with connect(url) as ws:
await hello(ws)
await ws.send(json.dumps(DELETABLE))
await send(ws, type="reply", id="r1", user="100", text="지워", via="chat", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r1")
await send(ws, type="reply_delivered", id="r1", ok=True) # the app posted the question
await send(ws, type="reply", id="r2", user="100", text="응, 지워", via="voice", speak=False)
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "chat_action")
await send(ws, type="chat_action_result", id=seen[-1]["id"], ok=True)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r2")
return seen[-1]
action = asyncio.run(with_worker(body, store=store, brain=brain))
assert (action["action"], action["channel"], action["message_id"]) == ("delete", "c1", "m1")
assert results == [("지워", True), ("응, 지워", True)]
@pytest.mark.unit
def test_a_yes_after_a_delete_question_they_never_got_does_not_delete(store):
import threading
release = threading.Event()
release.set()
brain, results = delete_tool_brain(release, question="") # the answer failed: nothing was asked
async def body(url):
async with connect(url) as ws:
await hello(ws)
await ws.send(json.dumps(DELETABLE))
await send(ws, type="reply", id="r1", user="100", text="지워", via="chat", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r1")
await send(ws, type="reply", id="r2", user="100", text="응, 지워", via="voice", speak=False)
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r2")
seen = asyncio.run(with_worker(body, store=store, brain=brain))
assert not any(isinstance(m, dict) and m.get("type") == "chat_action" for m in seen)
assert results == [("지워", True), ("응, 지워", False)]
def run_delete_then_yes(store, ask, after, question="지울까요?", tts=None, **kw):
"""The person asks to delete (``ask``: the reply message fields), ``after(ws)`` plays what happened
to the question, then they say "응, 지워" by voice. The chat actions that went out."""
import threading
release = threading.Event()
release.set()
brain, results = delete_tool_brain(release, question=question)
async def body(url):
async with connect(url) as ws:
await hello(ws)
await ws.send(json.dumps(DELETABLE))
await send(ws, type="reply", id="r1", user="100", text="지워", **ask)
await after(ws)
await send(ws, type="reply", id="r2", user="100", text="응, 지워", via="voice", speak=False)
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r2")
return [m for m in seen if isinstance(m, dict) and m.get("type") == "chat_action"]
actions = asyncio.run(with_worker(body, store=store, brain=brain, tts=tts, **kw))
return actions, results
def until(pred):
async def after(ws):
await recv_until(ws, pred)
return after
@pytest.mark.unit
def test_a_yes_after_a_question_that_could_not_be_spoken_does_not_delete(store):
# No TTS engine: the spoken question never reached them.
actions, results = run_delete_then_yes(
store, dict(via="voice", speak=True),
until(lambda m: isinstance(m, dict) and m.get("code") == "tts_unavailable"))
assert actions == [] and results == [("지워", True), ("응, 지워", False)]
@pytest.mark.unit
def test_a_yes_after_a_question_whose_speech_failed_does_not_delete(store):
actions, results = run_delete_then_yes(
store, dict(via="voice", speak=True),
until(lambda m: isinstance(m, dict) and m.get("type") == "tts_done" and m["failed"]),
question="FAIL 지울까요?", tts=FakeTts())
assert actions == [] and results == [("지워", True), ("응, 지워", False)]
@pytest.mark.unit
def test_a_yes_after_a_spoken_question_deletes(store):
import threading
release = threading.Event()
release.set()
brain, results = delete_tool_brain(release)
async def body(url):
async with connect(url) as ws:
await hello(ws)
await ws.send(json.dumps(DELETABLE))
await send(ws, type="reply", id="r1", user="100", text="지워", via="voice", speak=True)
done = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
assert not done[-1]["failed"] and not done[-1]["cancelled"]
await send(ws, type="reply", id="r2", user="100", text="응, 지워", via="voice", speak=False)
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "chat_action")
await send(ws, type="chat_action_result", id=seen[-1]["id"], ok=True)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r2")
return seen[-1]
action = asyncio.run(with_worker(body, store=store, brain=brain, tts=FakeTts()))
assert action["action"] == "delete" and results == [("지워", True), ("응, 지워", True)]
@pytest.mark.unit
def test_a_yes_after_a_chat_question_the_app_could_not_post_does_not_delete(store):
async def not_posted(ws):
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r1")
await send(ws, type="reply_delivered", id="r1", ok=False)
actions, results = run_delete_then_yes(store, dict(via="chat", speak=False), not_posted)
assert actions == [] and results == [("지워", True), ("응, 지워", False)]
@pytest.mark.unit
def test_a_chat_question_the_app_never_reports_counts_as_not_posted(store):
async def no_report(ws):
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r1")
await asyncio.sleep(0.3)
actions, results = run_delete_then_yes(store, dict(via="chat", speak=False), no_report,
delivery_timeout_s=0.1)
assert actions == [] and results == [("지워", True), ("응, 지워", False)]
class PreparingBrain(FakeBrain):
"""Each prepare waits until the test lets it finish."""
def __init__(self):
super().__init__()
import threading
self.prepared = []
self.release = threading.Event()
def prepare(self, user, text, chat):
self.prepared.append((user, text, chat))
self.release.wait(5)
@pytest.mark.unit
def test_prepare_runs_one_at_a_time_per_user_and_only_the_newest_different_text_waits(store):
brain = PreparingBrain()
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await send(ws, type="prepare", user="100", text="자비스 오늘")
await asyncio.sleep(0.2)
await send(ws, type="prepare", user="100", text="자비스 오늘 서울")
await send(ws, type="prepare", user="100", text="자비스 오늘 서울 날씨 어때")
await send(ws, type="prepare", user="200", text="자비스 안녕")
await asyncio.sleep(0.2)
assert sorted(brain.prepared) == [("100", "자비스 오늘", False), ("200", "자비스 안녕", False)]
brain.release.set()
await asyncio.sleep(0.3)
# About the same as the one just prepared: not prepared again.
await send(ws, type="prepare", user="100", text="자비스 오늘 서울 날씨 어때?")
await asyncio.sleep(0.3)
await send(ws, type="reply", id="r1", user="100", text="x", via="voice", speak=False)
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
seen = asyncio.run(with_worker(body, store=store, brain=brain))
assert [t for u, t, _ in brain.prepared if u == "100"] == ["자비스 오늘", "자비스 오늘 서울 날씨 어때"]
assert not [m for m in seen if isinstance(m, dict) and m.get("type") == "error"], "nothing is sent back"
@pytest.mark.unit
def test_a_brain_without_prepare_ignores_it(store):
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await send(ws, type="prepare", user="100", text="자비스 안녕")
await send(ws, type="reply", id="r1", user="100", text="hi", via="voice", speak=False)
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain()))
assert not [m for m in seen if isinstance(m, dict) and m.get("type") == "error"]
def _jarvis_brain(tmp_path, monkeypatch, plans):
"""A real JarvisBrain with the engine's judge and reply replaced: ``plans`` maps text to a pick."""
pytest.importorskip("mcp") # the real brain needs the jarvis dependencies (--brain)
from voice_worker import brain as brain_mod
import jarvis.reply.engine as engine
cfg = tmp_path / "config.json"
cfg.write_text(json.dumps({"db_path": str(tmp_path / "jarvis.db")}))
monkeypatch.setenv("JARVIS_CONFIG_PATH", str(cfg))
seen = {"judged": [], "prepared": []}
def fake_prepare(cfg, text, mem, chat_available, schedule_available=False, notes_available=False,
screen_available=False, pc_available=False):
# the reply it is made for has these tools
assert schedule_available and notes_available and screen_available and pc_available
seen["judged"].append((text, chat_available))
return plans.get(text)
def fake_engine(db, cfg, tts, text, mem, prepared_tools=None, intent_judge=False, prepared_done=None, **kw):
seen["prepared"].append(prepared_tools)
seen["intent_judge"] = intent_judge
seen.setdefault("done", []).append(prepared_done() if prepared_done is not None else "not passed")
mem.add_message("user", text)
mem.add_message("assistant", "ok")
return "ok"
def fake_done(cfg, text, mem):
seen.setdefault("done_asked", []).append(text)
return True
monkeypatch.setattr(engine, "prepare_intent", fake_prepare)
monkeypatch.setattr(engine, "prepare_done_by_user", fake_done)
monkeypatch.setattr(engine, "run_reply_engine", fake_engine)
return brain_mod.JarvisBrain(), seen
@pytest.mark.unit
def test_a_reply_uses_the_pick_prepared_for_about_the_same_text(tmp_path, monkeypatch):
from jarvis.memory.history import HistoryContext
plan = ["getWeather"]
b, seen = _jarvis_brain(tmp_path, monkeypatch, {"자비스 오늘 서울 날씨 어": plan})
b.prepare("100", "자비스 오늘 서울 날씨 어", True)
assert seen["judged"] == [("자비스 오늘 서울 날씨 어", True)]
b.reply("100", "자비스, 오늘 서울 날씨 어때?", HistoryContext())
assert seen["prepared"] == [plan] and seen["intent_judge"] is True
# Used once: the next reply judges for itself.
b.reply("100", "자비스, 오늘 서울 날씨 어때?", HistoryContext())
assert seen["prepared"][-1] is None
@pytest.mark.unit
def test_a_prepared_pick_is_not_used_for_other_words_another_person_or_a_changed_dialogue(tmp_path, monkeypatch):
from jarvis.memory.history import HistoryContext
plan = ["webSearch"]
b, seen = _jarvis_brain(tmp_path, monkeypatch, {"자비스 오늘 서울 날씨 어때": plan})
b.prepare("100", "자비스 오늘 서울 날씨 어때", False)
b.reply("100", "자비스, 내일 회의 몇 시지?", HistoryContext())
b.prepare("100", "자비스 오늘 서울 날씨 어때", False)
b.reply("200", "자비스 오늘 서울 날씨 어때", HistoryContext())
assert seen["prepared"] == [None, None]
b.prepare("100", "자비스 오늘 서울 날씨 어때", False)
b._memory("100").add_message("user", "something said meanwhile")
b.reply("100", "자비스 오늘 서울 날씨 어때", HistoryContext())
assert seen["prepared"][-1] is None
@pytest.mark.unit
def test_a_reply_uses_the_done_by_user_judgement_prepared_for_its_text(tmp_path, monkeypatch):
"""The done-by-user guard's first question is asked while the user still speaks, so a follow-up
turn does not wait for it after the answer (reply.spec.md, "Earlier answers after the user did it")."""
from jarvis.memory.history import HistoryContext
b, seen = _jarvis_brain(tmp_path, monkeypatch, {"자비스 속도 조절했으니 말해 봐": []})
b.prepare("100", "자비스 속도 조절했으니 말해 봐", False)
assert seen["done_asked"] == ["자비스 속도 조절했으니 말해 봐"]
b.reply("100", "자비스, 속도 조절했으니 말해 봐", HistoryContext())
assert seen["done"] == [True]
# Not prepared: the guard asks for itself.
b.reply("100", "자비스, 속도 조절했으니 말해 봐", HistoryContext())
assert seen["done"][-1] == "not passed"
@pytest.mark.unit
def test_a_slow_done_judgement_does_not_hold_up_the_prepared_pick(tmp_path, monkeypatch):
"""The reply starts with the prepared tools at once; only the guard waits for the judgement."""
import threading
import time as _time
from jarvis.memory.history import HistoryContext
import jarvis.reply.engine as engine
b, seen = _jarvis_brain(tmp_path, monkeypatch, {"자비스 날씨 어때": ["getWeather"]})
release = threading.Event()
def slow_done(cfg, text, mem):
release.wait(5)
return False
monkeypatch.setattr(engine, "prepare_done_by_user", slow_done)
started = {}
def fake_engine(db, cfg, tts, text, mem, prepared_tools=None, prepared_done=None, **kw):
started["tools"] = prepared_tools
release.set() # the answer is being made while the judgement finishes
started["done"] = prepared_done()
return "ok"
monkeypatch.setattr(engine, "run_reply_engine", fake_engine)
t = threading.Thread(target=b.prepare, args=("100", "자비스 날씨 어때", False))
t.start()
_time.sleep(0.1)
t0 = _time.monotonic()
b.reply("100", "자비스 날씨 어때?", HistoryContext())
t.join()
assert started == {"tools": ["getWeather"], "done": False}
assert _time.monotonic() - t0 < 2
@pytest.mark.unit
def test_a_reply_waits_for_a_pick_still_being_prepared_for_its_text(tmp_path, monkeypatch):
import threading
import time as _time
from jarvis.memory.history import HistoryContext
plan = [] # "no tools" is a pick too
b, seen = _jarvis_brain(tmp_path, monkeypatch, {})
import jarvis.reply.engine as engine
def slow_prepare(cfg, text, mem, chat_available, schedule_available=False, notes_available=False,
screen_available=False, pc_available=False):
_time.sleep(0.4)
return plan
monkeypatch.setattr(engine, "prepare_intent", slow_prepare)
t = threading.Thread(target=b.prepare, args=("100", "자비스 날씨 어때", False))
t.start()
_time.sleep(0.1)
b.reply("100", "자비스 날씨 어때?", HistoryContext())
t.join()
assert seen["prepared"] == [plan]
@pytest.mark.unit
def test_worker_logs_name_their_service_so_the_app_puts_them_in_its_tab(store, monkeypatch, capfd):
"""The app's log screen puts a worker line in the stt, llm or tts tab by its category
(app/src/admin.spec.md, "Logs"); the worker must send those categories."""
from voice_worker import log as worker_log
monkeypatch.setattr(worker_log._jarvis_debug, "_is_debug_enabled", lambda: True)
def broken(user, text):
raise RuntimeError("model down")
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await say(ws, "100", 0.9)
await send(ws, type="reply", id="r1", user="100", text="hi", via="voice", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
await send(ws, type="tts", id="t1", text="안녕하세요.", more=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
asyncio.run(with_worker(body, store=store, brain=FakeBrain(answer=broken), tts=FakeTts()))
err = capfd.readouterr().err
for cat, what in (("stt", "utterance end"), ("llm", "brain reply r1 failed"), ("tts", "tts sentence id=t1")):
assert any(line.startswith(f"[{cat:^10}]") and what in line for line in err.splitlines()), (cat, err[-2000:])
async def drain(ws, seconds):
"""Every message that arrives within `seconds`."""
out, end = [], asyncio.get_running_loop().time() + seconds
while (left := end - asyncio.get_running_loop().time()) > 0:
try:
raw = await asyncio.wait_for(ws.recv(), left)
except asyncio.TimeoutError:
break
out.append(raw if isinstance(raw, bytes) else json.loads(raw))
return out
@pytest.mark.unit
def test_an_llm_switch_waits_for_the_answer_in_progress_then_holds_new_ones_with_one_notice(store):
"""voice_worker.spec.md, "LLM switches": the app switches the LLM only once the brain is idle;
requests meanwhile wait, the first spoken one hears the notice once, and all are answered once
the LLM is ready."""
import threading
release = threading.Event()
def slow_first(user, text):
if text == "first":
release.wait(10)
return f"answer for {text}"
tts = FakeTts()
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="reply", id="r1", user="100", text="first", via="voice", speak=True)
await asyncio.sleep(0.1)
await send(ws, type="llm", state="switching", notice="잠깐만요, 바꾸는 중이에요")
# Not idle while the first answer is being written.
seen = []
try:
seen += await asyncio.wait_for(recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "llm_idle"), 0.3)
idle_early = True
except asyncio.TimeoutError:
idle_early = False
release.set()
seen += await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "llm_idle")
# Two requests while switching: they wait; only the first spoken one hears the notice.
await send(ws, type="reply", id="r2", user="200", text="second", via="voice", speak=True)
await send(ws, type="reply", id="r3", user="300", text="third", via="voice", speak=True)
seen += await drain(ws, 0.4)
answered_while_switching = [m["id"] for m in seen if isinstance(m, dict) and m.get("type") == "reply_text"]
await send(ws, type="llm", state="ready")
seen += await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done" and m["id"] == "r3")
return idle_early, answered_while_switching, seen
brain = FakeBrain(slow_first)
idle_early, early, seen = asyncio.run(with_worker(body, store=store, brain=brain, tts=tts))
assert not idle_early, "idle only after the answer in progress"
assert "r2" not in early and "r3" not in early, "nothing new is answered while switching"
assert [c[1] for c in brain.calls] == ["first", "second", "third"]
assert tts.texts.count("잠깐만요, 바꾸는 중이에요") == 1, tts.texts
done = {m["id"]: m for m in seen if isinstance(m, dict) and m.get("type") == "tts_done"}
assert not done["r2"]["failed"] and not done["r3"]["failed"]
# r2 heard the notice, then its answer, under its own id.
assert tts.texts.index("잠깐만요, 바꾸는 중이에요") < tts.texts.index("answer for second")
@pytest.mark.unit
def test_while_paused_the_brain_answers_nothing(store):
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="llm", state="switching")
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "llm_idle")
await send(ws, type="llm", state="paused")
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
brain = FakeBrain()
seen = asyncio.run(with_worker(body, store=store, brain=brain))
reply = [m for m in seen if isinstance(m, dict) and m.get("type") == "reply_text"][0]
assert reply["failed"] and brain.calls == []
@pytest.mark.unit
def test_an_unknown_llm_state_is_refused(store):
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="llm", state="maybe")
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "error")
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain()))
assert [m for m in seen if isinstance(m, dict) and m.get("type") == "error"][0]["code"] == "bad_message"
@pytest.mark.unit
def test_llm_idle_names_the_switch_it_answers(store):
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="llm", state="switching", id=7)
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "llm_idle")
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain()))
assert [m for m in seen if isinstance(m, dict) and m.get("type") == "llm_idle"] == [{"type": "llm_idle", "id": 7}]
@pytest.mark.unit
def test_a_request_that_waits_too_long_for_a_switch_fails_without_the_brain(store, monkeypatch):
import voice_worker.server as server
monkeypatch.setattr(server, "LLM_WAIT_S", 0.3)
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="llm", state="switching", id=1)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "llm_idle")
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
brain = FakeBrain()
seen = asyncio.run(with_worker(body, store=store, brain=brain))
reply = [m for m in seen if isinstance(m, dict) and m.get("type") == "reply_text"][0]
assert reply["failed"] and brain.calls == [], "the LLM may be away: the brain is not called"
@pytest.mark.unit
def test_in_a_game_no_partials_are_transcribed_while_an_answer_is_written(store):
"""voice_worker.spec.md, "LLM switches": `game` on ready holds partial passes while the brain
writes an answer (M0 gamemode_tune); without a game they go on."""
import threading
from tests.test_voice_worker_server import stream
for game in (True, False):
release = threading.Event()
def slow(user, text):
release.wait(10)
return f"answer for {text}"
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partial_interval_ms": 100})
await send(ws, type="llm", state="ready", game=game)
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
await asyncio.sleep(0.1)
await stream(ws, "200", 0.9, 800, realtime=True)
seen = await drain(ws, 0.8)
release.set()
seen += await drain(ws, 0.5)
return seen
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain(slow)))
partials = [m for m in seen if isinstance(m, dict) and m.get("type") == "partial"]
replies = [i for i, m in enumerate(seen) if isinstance(m, dict) and m.get("type") == "reply_text"]
first_partial = next((i for i, m in enumerate(seen) if isinstance(m, dict) and m.get("type") == "partial"), None)
if game:
assert first_partial is None or (replies and first_partial > replies[0]), "no partial while answering"
else:
assert partials and replies and first_partial < replies[0], "partials go on without a game"
@pytest.mark.unit
@pytest.mark.parametrize("via, channel", [("voice", None), ("chat", "c1")])
def test_a_schedule_action_goes_to_the_app_for_the_asker_and_where_they_asked(store, via, channel):
results = []
brain = FakeBrain(lambda u, t: results.append(brain.schedule.act("add", kind="remind", text="회의",
at=1_900_000_000.0, repeat="none")) or "ok")
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="r1", user="100", text="q", via=via, speak=False,
**({"channel": channel} if channel else {}))
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "schedule_action")
action = seen[-1]
await send(ws, type="schedule_action_result", id=action["id"], ok=True, item={"id": "7", "text": "회의"})
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
return action
action = asyncio.run(with_worker(body, store=store, brain=brain))
assert {k: v for k, v in action.items() if k not in ("type", "id")} == {
"action": "add", "user": "100", "via": via, **({"channel": channel} if channel else {}),
"kind": "remind", "text": "회의", "at": 1_900_000_000.0, "repeat": "none"}
assert results == [{"ok": True, "item": {"id": "7", "text": "회의"}}]
@pytest.mark.unit
def test_a_schedule_action_the_app_refuses_or_never_answers_fails(store):
results = []
brain = FakeBrain(lambda u, t: results.extend([brain.schedule.act("list"), brain.schedule.act("list")]) or "ok")
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="r1", user="100", text="q", via="voice", speak=False)
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "schedule_action")
await send(ws, type="schedule_action_result", id=seen[-1]["id"], ok=False, error="no")
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
asyncio.run(with_worker(body, store=store, brain=brain, chat_timeout_s=0.3))
assert results == [{"ok": False, "error": "no"}, {"ok": False, "error": "the app did not answer in time"}]
@pytest.mark.unit
def test_a_scheduled_task_is_answered_without_schedule_or_chat_and_is_no_turn(store):
brain = FakeBrain()
async def body(url):
async with connect(url) as ws:
await hello(ws)
await ws.send(json.dumps(CHANNELS))
await send(ws, type="reply", id="t1", user="100", text="날씨 알려 줘", via="task")
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
await send(ws, type="reply", id="t2", user="100", text="x", via="task", speak=True)
seen += await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "error")
return seen
worker_turns = {}
async def run():
worker = VoiceWorker(TOKEN, LevelTranscriber(), AmplitudeVad, None, history=store, brain=brain)
server = await worker.serve("127.0.0.1", 0)
try:
seen = await asyncio.wait_for(body(f"ws://127.0.0.1:{server.sockets[0].getsockname()[1]}"), 15)
worker_turns.update(worker.reply_turns)
return seen
finally:
server.close()
await server.wait_closed()
await worker.stop()
seen = asyncio.run(run())
answer = next(m for m in seen if isinstance(m, dict) and m.get("type") == "reply_text")
assert answer == {"type": "reply_text", "id": "t1", "text": "answer for 날씨 알려 줘", "failed": False}
assert brain.task is True and brain.chat is None and brain.schedule is None
assert worker_turns == {} # not a turn: a delete asked before it is still confirmable
assert any(isinstance(m, dict) and m.get("type") == "error" and m.get("id") == "t2" for m in seen)
@pytest.mark.unit
def test_a_scheduled_task_runs_with_the_read_only_tools_and_no_mcp(tmp_path, monkeypatch):
pytest.importorskip("mcp")
from jarvis.memory.history import HistoryContext
from jarvis.reply import turn_state
from jarvis.reply.engine import _offered_builtins
from voice_worker import brain as brain_mod
cfg = tmp_path / "config.json"
cfg.write_text(json.dumps({"db_path": str(tmp_path / "jarvis.db"), "disabled_tools": ["getWeather"],
"mcps": {"x": {"transport": "stdio", "command": "nope"}}}))
monkeypatch.setenv("JARVIS_CONFIG_PATH", str(cfg))
monkeypatch.setattr(brain_mod, "_connect_mcp", lambda mcps: None)
monkeypatch.setenv("JARVIS_COLLECT_PATH", str(tmp_path / "collect.json")) # the collection exists
seen = []
def fake_engine(db, cfg, tts, text, mem, **kw):
seen.append((set(_offered_builtins(cfg, turn_state.get_chat() is not None,
turn_state.get_schedule() is not None)), cfg.mcps, mem))
return "ok"
import jarvis.reply.engine as engine
monkeypatch.setattr(engine, "run_reply_engine", fake_engine)
b = brain_mod.JarvisBrain()
b.reply("100", "q", HistoryContext(), chat=object(), schedule=object())
b.reply("100", "q", HistoryContext(), chat=object(), schedule=object(), task=True)
(normal, normal_mcps, normal_mem), (task, task_mcps, task_mem) = seen
assert {"schedule", "chatChannels", "logMeal"} <= normal and normal_mcps
# Only looking things up, and a feature switched off stays off.
assert task == set(brain_mod.TASK_TOOLS) - {"getWeather"} and task_mcps == {}
assert task_mem is not normal_mem # not a turn of their conversation
@pytest.mark.unit
def test_the_askers_notes_from_the_app_reach_the_brain_and_note_actions_go_back(store):
results = []
brain = FakeBrain(lambda u, t: results.append(brain.notes_backend.act("add", text="반말로 해")) or "ok")
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", channel="c1", speak=False,
notes=["섭씨로"])
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "note_action")
action = seen[-1]
await send(ws, type="note_action_result", id=action["id"], ok=True, item={"id": "3", "text": "반말로 해"})
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
await send(ws, type="reply", id="r2", user="100", text="q", via="voice", speak=False, notes=[1])
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "error")
return action, seen[-1]
action, error = asyncio.run(with_worker(body, store=store, brain=brain))
assert brain.notes == ["섭씨로"]
assert {k: v for k, v in action.items() if k not in ("type", "id")} == {
"action": "add", "user": "100", "via": "chat", "channel": "c1", "text": "반말로 해"}
assert results == [{"ok": True, "item": {"id": "3", "text": "반말로 해"}}]
assert error["code"] == "bad_message" and error["id"] == "r2"
@pytest.mark.unit
def test_a_scheduled_task_gets_the_persons_notes_but_no_notes_tool(store):
brain = FakeBrain()
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="t1", user="100", text="날씨", via="task", notes=["섭씨로"])
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
asyncio.run(with_worker(body, store=store, brain=brain))
assert brain.notes == ["섭씨로"] and brain.notes_backend is None and brain.task is True
@pytest.mark.unit
def test_a_screen_action_goes_to_the_app_for_the_asker_and_its_pictures_come_back(store):
results = []
brain = FakeBrain(lambda u, t: results.append(brain.screen.act("capture", target="monitor", monitor=2,
detail="normal")) or "ok")
picture = "A" * (2 * 2 ** 20) # pictures make messages larger than the other actions'
async def body(url):
async with connect(url, max_size=None) as ws:
await hello(ws)
await send(ws, type="reply", id="r1", user="100", text="q", via="voice", speak=False)
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "screen_action")
action = seen[-1]
await send(ws, type="screen_action_result", id=action["id"], ok=True,
item={"what": "모니터 2", "width": 1280, "height": 720, "parts": 1}, images=[picture, 3])
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
await send(ws, type="reply", id="t1", user="100", text="q", via="task")
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
return action
action = asyncio.run(with_worker(body, store=store, brain=brain))
assert {k: v for k, v in action.items() if k not in ("type", "id")} == {
"action": "capture", "user": "100", "via": "voice", "target": "monitor", "monitor": 2, "detail": "normal"}
assert results == [{"ok": True, "item": {"what": "모니터 2", "width": 1280, "height": 720, "parts": 1},
"images": [picture]}]
assert brain.screen is None # a scheduled task sees no screen
@pytest.mark.unit
def test_a_pc_action_goes_to_the_app_and_an_ask_answer_keeps_what_to_ask(store):
"""pc_control.spec.md: the app answers ask with the kinds and the window; the worker keeps the
asker's requests awaiting consent between their replies and numbers the replies."""
results, seen_pc = [], []
def answer(u, t):
seen_pc.append((brain.pc.turn, brain.pc.state))
if t == "q1":
results.append(brain.pc.act("click", ref="@s1:e1"))
brain.pc.state["pending"] = {"w1": {"turn": brain.pc.turn}}
return "ok"
brain = FakeBrain(answer)
window = {"id": "w1", "program": "CalculatorApp.exe", "title": "계산기"}
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="r1", user="100", text="q1", via="voice", speak=False)
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "pc_action")
action = seen[-1]
await send(ws, type="pc_action_result", id=action["id"], ok=False, ask=True, kinds=["click"],
window=window)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
await send(ws, type="reply", id="r2", user="100", text="q2", via="voice", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
await send(ws, type="reply", id="t1", user="100", text="q", via="task")
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
return action
action = asyncio.run(with_worker(body, store=store, brain=brain))
assert {k: v for k, v in action.items() if k not in ("type", "id")} == {
"action": "click", "user": "100", "via": "voice", "ref": "@s1:e1"}
assert results == [{"ok": False, "ask": True, "kinds": ["click"], "window": window}]
(t1, s1), (t2, s2) = seen_pc
assert t2 == t1 + 1 and s2 is s1 and s2["pending"] == {"w1": {"turn": t1}}
assert brain.pc is None # a scheduled task operates nothing
@pytest.mark.unit
def test_pc_and_screen_actions_wait_longer_than_the_other_app_actions(store):
"""Starting a program and reading its new window takes the app up to about 20 s (admin.spec.md,
"PC control"), and a screen share is waited for up to 30 s while it is popped out ("Screen
viewing"): those actions are not given up after the chat actions' wait."""
results = []
brain = FakeBrain(lambda u, t: results.append((brain.notes_backend.act("list"),
brain.screen.act("capture", target="share"),
brain.pc.act("run", program="calc"))) or "ok")
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="r1", user="100", text="q", via="voice", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "note_action")
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "screen_action")
await asyncio.sleep(0.6) # longer than chat_timeout_s
await send(ws, type="screen_action_result", id=seen[-1]["id"], ok=True, item={"what": "화면공유"},
images=["x"])
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "pc_action")
await asyncio.sleep(0.6)
await send(ws, type="pc_action_result", id=seen[-1]["id"], ok=True, item={"window": {"id": "w1"}})
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
asyncio.run(with_worker(body, store=store, brain=brain, chat_timeout_s=0.3))
(note, screen, pc), = results
assert not note["ok"] and screen["ok"] and pc == {"ok": True, "item": {"window": {"id": "w1"}}}
@pytest.mark.unit
def test_the_judgement_is_told_who_is_in_the_channel(monkeypatch):
import voice_worker.judges as judges
asked = []
monkeypatch.setattr(judges, "call_llm_direct", lambda b, m, system, u, **k: asked.append(system) or "yes")
for people, callers in ((None, None), (2, 2), (5, 1)):
judges.is_addressed("u", "m", ["자비스"], "x", people=people, callers=callers)
unknown, pair, group = asked
assert "Several people" in unknown and "anything unclear" in unknown, "an app that does not say: as before"
assert "one other person" in pair and "unclear, answer yes" in pair
assert "5 people" in group and "Only 1 of the 5" in group and "anything unclear" in group
@pytest.mark.unit
def test_each_answer_logs_the_least_graphics_memory_free_while_it_was_written(store, monkeypatch, capfd):
"""voice_worker.spec.md, "Debug logging": an answer (and a scheduled task) logs the least free
graphics memory read while the LLM wrote it, without debug logging and without its text, so a
slow answer can be matched to a full card."""
import time as _time
from voice_worker import server as worker_server
monkeypatch.setattr(worker_server, "CARD_EVERY_S", 0.02)
# Free memory as nvidia-smi reports it (the driver's reserved share is not free).
readings = iter([7000, 859, 3000])
last = [3000]
def card():
last[0] = next(readings, last[0])
return last[0], 16303
def slow(user, text):
_time.sleep(0.3)
return f"secret answer for {text}"
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="r1", user="100", text="private question", via="chat", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
await send(ws, type="reply", id="t1", user="100", text="private task", via="task")
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text" and m["id"] == "t1")
asyncio.run(with_worker(body, store=store, brain=FakeBrain(slow), card=card))
lines = [x for x in capfd.readouterr().err.splitlines() if x.startswith(f"[{'llm':^10}]") and "free" in x]
reply = [x for x in lines if "answer r1" in x]
task = [x for x in lines if "task t1" in x]
assert len(reply) == 1 and len(task) == 1, lines
assert "least 859 MiB free of 16303 MiB" in reply[0], reply
assert "private" not in "".join(lines) and "secret" not in "".join(lines)
@pytest.mark.unit
def test_without_a_card_reading_an_answer_logs_no_card_line(store, capfd):
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
asyncio.run(with_worker(body, store=store, brain=FakeBrain(), card=lambda: None))
assert "MiB free" not in capfd.readouterr().err
@pytest.mark.unit
def test_an_answer_done_before_the_first_card_reading_is_logged_once_the_reading_ends(store, capfd):
"""A short answer is not held for the card, and still gets its line once the reading ends."""
import threading
reading = threading.Event()
def card():
reading.wait(5)
return 2048, 16303
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
before = capfd.readouterr().err
reading.set()
await asyncio.sleep(0.3)
return before
before = asyncio.run(with_worker(body, store=store, brain=FakeBrain(), card=card))
after = capfd.readouterr().err
assert "answer r1 card" not in before, "the answer did not wait for the card"
assert "answer r1 card: least 2048 MiB free of 16303 MiB (1 readings" in after, after[-1500:]