v1's tts_engine default (English-only Piper) made the reply engine force English, so Korean questions got English answers. The v2 brain speaks through the worker's MeloTTS and chat answers are not spoken, so it runs with tts_engine=melo; an output_language lock still wins. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
962 lines
42 KiB
Python
962 lines
42 KiB
Python
"""Voice worker history and brain messages (voice_worker.spec.md, "History and brain").
|
|
|
|
Over a real local WebSocket with the fake STT/VAD/TTS engines of test_voice_worker_server and a
|
|
fake brain that records the history context it was given. Behaviours: presence and finals reach
|
|
the history store, chat events are kept, a reply answers with the asker's own scope, a voice answer
|
|
is stored and can be spoken, and the worker reports when it has no brain.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from jarvis.memory.history import HistoryStore, Scope
|
|
from voice_worker.protocol import decode_tts_frame
|
|
from voice_worker.server import VoiceWorker
|
|
|
|
from tests.test_voice_worker_server import TOKEN, AmplitudeVad, FakeTts, LevelTranscriber, hello, recv_until, stream
|
|
|
|
websockets = pytest.importorskip("websockets")
|
|
from websockets.asyncio.client import connect # noqa: E402
|
|
|
|
|
|
class FakeBrain:
|
|
def __init__(self, answer=lambda user, text: f"answer for {text}"):
|
|
self.answer = answer
|
|
self.calls = []
|
|
|
|
def reply(self, user, text, history, chat=None):
|
|
self.calls.append((user, text, history))
|
|
self.chat = chat
|
|
return self.answer(user, text)
|
|
|
|
def addressed(self, user, text, names):
|
|
self.calls.append(("addressed", user, text, names))
|
|
return "jarvis" in text
|
|
|
|
def rank(self, candidates):
|
|
self.calls.append(("rank", [c["id"] for c in candidates]))
|
|
return [c["id"] for c in reversed(candidates)]
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path):
|
|
s = HistoryStore(str(tmp_path / "history.sqlite"))
|
|
yield s
|
|
s.close()
|
|
|
|
|
|
async def with_worker(body, store=None, brain=None, tts=None, **kw):
|
|
worker = VoiceWorker(TOKEN, LevelTranscriber(), AmplitudeVad, tts, history=store, brain=brain, **kw)
|
|
server = await worker.serve("127.0.0.1", 0)
|
|
port = server.sockets[0].getsockname()[1]
|
|
try:
|
|
return await asyncio.wait_for(body(f"ws://127.0.0.1:{port}"), 15)
|
|
finally:
|
|
server.close()
|
|
await server.wait_closed()
|
|
await worker.stop()
|
|
|
|
|
|
async def send(ws, **msg):
|
|
await ws.send(json.dumps(msg))
|
|
|
|
|
|
async def say(ws, speaker, level):
|
|
await stream(ws, speaker, level, 600)
|
|
await send(ws, type="speaking_end", speaker=speaker)
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final" and m["speaker"] == speaker)
|
|
|
|
|
|
def texts(records):
|
|
return [r.text for r in records]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_finals_in_a_voice_channel_are_stored_under_the_speakers_name(store):
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws, stt={"partials": False})
|
|
await say(ws, "100", 0.9) # before Jarvis is in a channel: not stored
|
|
await send(ws, type="voice_join", channel="vc1")
|
|
await send(ws, type="member_join", user="100", name="Alice")
|
|
await say(ws, "100", 0.6)
|
|
|
|
asyncio.run(with_worker(body, store=store))
|
|
recs = store.records_of("100")
|
|
assert texts(recs) == ["level 0.6"]
|
|
assert recs[0].author_name == "Alice" and recs[0].channel_id == "vc1" and recs[0].kind == "voice"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_reply_answers_with_the_askers_own_scope_and_stores_the_voice_answer(store):
|
|
brain = FakeBrain()
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws, stt={"partials": False})
|
|
await send(ws, type="voice_join", channel="vc1")
|
|
await send(ws, type="member_join", user="100", name="Alice")
|
|
await say(ws, "100", 0.9)
|
|
await send(ws, type="member_join", user="200", name="Bob")
|
|
await say(ws, "200", 0.6)
|
|
await send(ws, type="reply", id="r1", user="100", text="what did we say", via="voice", speak=False)
|
|
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
|
|
await send(ws, type="reply", id="r2", user="200", text="and me", via="voice", speak=False)
|
|
seen += await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
|
|
return seen
|
|
|
|
seen = asyncio.run(with_worker(body, store=store, brain=brain))
|
|
answers = [m for m in seen if isinstance(m, dict) and m.get("type") == "reply_text"]
|
|
assert answers[0] == {"type": "reply_text", "id": "r1", "text": "answer for what did we say", "failed": False}
|
|
# Alice was there when Bob spoke, so she sees both; Bob joined after Alice spoke.
|
|
alice_ctx, bob_ctx = brain.calls[0][2], brain.calls[1][2]
|
|
assert any("Alice: level 0.9" in x for x in alice_ctx.recent)
|
|
assert any("Bob: level 0.6" in x for x in alice_ctx.recent)
|
|
assert not any("Alice: level 0.9" in x for x in bob_ctx.recent)
|
|
# The voice answer is Alice's own record now.
|
|
assert "answer for what did we say" in texts(store.records_of("100"))
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_spoken_reply_comes_back_as_tts_under_the_same_id(store):
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await send(ws, type="voice_join", channel="vc1")
|
|
await send(ws, type="reply", id="r9", user="100", text="hello", via="voice", speak=True)
|
|
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
|
|
|
|
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain(), tts=FakeTts()))
|
|
frames = [decode_tts_frame(m) for m in seen if isinstance(m, bytes)]
|
|
assert frames and all(rid == "r9" for rid, _, _ in frames)
|
|
done = [m for m in seen if isinstance(m, dict) and m.get("type") == "tts_done"][0]
|
|
assert done["id"] == "r9" and not done["failed"]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_chat_answers_are_not_stored_by_the_worker_but_reported_messages_are(store):
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await send(ws, type="reply", id="c1", user="100", text="hi", via="chat", speak=False)
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
|
|
await send(ws, type="chat_message", channel="ch1", message_id="m1", author="100", name="Alice",
|
|
text="first", ts=1_800_000_000.0)
|
|
await send(ws, type="chat_message", channel="ch1", message_id="m2", author="900", name="Jarvis",
|
|
text="answer for hi", ts=1_800_000_001.0, to="100")
|
|
await send(ws, type="chat_edit", channel="ch1", message_id="m1", text="first, edited")
|
|
await send(ws, type="chat_message", channel="ch1", message_id="m3", author="100", name="Alice",
|
|
text="gone soon", ts=1_800_000_002.0)
|
|
await send(ws, type="chat_delete", channel="ch1", message_id="m3")
|
|
# A round trip makes sure the worker handled everything above.
|
|
await send(ws, type="reply", id="c2", user="100", text="x", via="chat", speak=False)
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "c2")
|
|
|
|
asyncio.run(with_worker(body, store=store, brain=FakeBrain()))
|
|
assert texts(store.records_of("100")) == ["first, edited", "answer for hi"]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_leaving_the_voice_channel_stops_storing_and_closes_presence(store):
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws, stt={"partials": False})
|
|
await send(ws, type="voice_join", channel="vc1")
|
|
await send(ws, type="member_join", user="100", name="Alice")
|
|
await send(ws, type="voice_leave")
|
|
await say(ws, "200", 0.6)
|
|
|
|
asyncio.run(with_worker(body, store=store))
|
|
assert store.records_of("200") == []
|
|
assert store.recent("100", now=4_000_000_000, scope=Scope(), minutes=10 ** 9) == []
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_memory_off_gives_the_brain_no_history(store):
|
|
brain = FakeBrain()
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws, stt={"partials": False}, memory={"off": True})
|
|
await send(ws, type="voice_join", channel="vc1")
|
|
await say(ws, "100", 0.9)
|
|
await send(ws, type="reply", id="r1", user="100", text="q", via="voice", speak=False)
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
|
|
await send(ws, type="memory_config", off=False)
|
|
await send(ws, type="reply", id="r2", user="100", text="q", via="voice", speak=False)
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r2")
|
|
|
|
asyncio.run(with_worker(body, store=store, brain=brain))
|
|
off_ctx, on_ctx = brain.calls[0][2], brain.calls[1][2]
|
|
assert off_ctx.recent == [] and off_ctx.matches == []
|
|
assert any("level 0.9" in x for x in on_ctx.recent)
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("answer", [lambda u, t: None, lambda u, t: 1 / 0])
|
|
def test_a_failed_brain_answer_is_reported(store, answer):
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await send(ws, type="reply", id="r1", user="100", text="q", via="voice", speak=True)
|
|
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
|
|
|
|
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain(answer), tts=FakeTts()))
|
|
assert seen[-1] == {"type": "reply_text", "id": "r1", "text": "", "failed": True}
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_without_a_brain_reply_is_an_error():
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await send(ws, type="voice_join", channel="vc1")
|
|
await send(ws, type="reply", id="r1", user="100", text="q", via="voice", speak=False)
|
|
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "error")
|
|
|
|
seen = asyncio.run(with_worker(body))
|
|
assert seen[-1]["code"] == "brain_unavailable" and seen[-1]["id"] == "r1"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_malformed_history_messages_are_rejected(store):
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await send(ws, type="member_join", user=5)
|
|
await send(ws, type="reply", id="r1", user="100", via="radio")
|
|
return [await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "error")
|
|
for _ in range(2)]
|
|
|
|
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain()))
|
|
assert [s[-1]["code"] for s in seen] == ["bad_message", "bad_message"]
|
|
|
|
|
|
class SlowTranscriber(LevelTranscriber):
|
|
def __init__(self, delay):
|
|
super().__init__()
|
|
self.delay = delay
|
|
|
|
def transcribe(self, clips, opts):
|
|
import time as _t
|
|
_t.sleep(self.delay)
|
|
return super().transcribe(clips, opts)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_slow_final_keeps_the_channel_time_and_name_of_when_it_was_said(store):
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws, stt={"partials": False})
|
|
await send(ws, type="voice_join", channel="vc1")
|
|
await send(ws, type="member_join", user="100", name="Alice")
|
|
await stream(ws, "100", 0.9, 600)
|
|
await send(ws, type="speaking_end", speaker="100")
|
|
# While the model is still decoding: a late joiner arrives, then Jarvis moves.
|
|
await asyncio.sleep(0.15)
|
|
await send(ws, type="member_join", user="300", name="Late")
|
|
await send(ws, type="member_join", user="100", name="Renamed")
|
|
await send(ws, type="voice_join", channel="vc2")
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final")
|
|
|
|
worker = VoiceWorker(TOKEN, SlowTranscriber(0.6), AmplitudeVad, None, history=store)
|
|
|
|
async def run():
|
|
server = await worker.serve("127.0.0.1", 0)
|
|
port = server.sockets[0].getsockname()[1]
|
|
try:
|
|
await asyncio.wait_for(body(f"ws://127.0.0.1:{port}"), 15)
|
|
finally:
|
|
server.close()
|
|
await server.wait_closed()
|
|
await worker.stop()
|
|
|
|
asyncio.run(run())
|
|
recs = store.records_of("100")
|
|
assert [(r.channel_id, r.author_name, r.text) for r in recs] == [("vc1", "Alice", "level 0.9")]
|
|
# The late joiner was not there when it was said.
|
|
assert store.recent("300", now=4_000_000_000, scope=Scope(), minutes=10 ** 9) == []
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_presence_left_open_by_a_previous_run_is_closed_at_start(store):
|
|
store.presence_join("old", "vc1", 1_000.0) # the worker was killed with "old" still present
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws, stt={"partials": False})
|
|
await send(ws, type="voice_join", channel="vc1")
|
|
await send(ws, type="member_join", user="100", name="Alice")
|
|
await say(ws, "100", 0.9)
|
|
|
|
asyncio.run(with_worker(body, store=store))
|
|
assert texts(store.recent("100", now=4_000_000_000, scope=Scope(), minutes=10 ** 9)) == ["level 0.9"]
|
|
assert store.recent("old", now=4_000_000_000, scope=Scope(), minutes=10 ** 9) == []
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_the_brain_prints_no_text_from_its_thread(tmp_path, monkeypatch, capfd):
|
|
pytest.importorskip("mcp") # the real brain needs the jarvis dependencies (--brain)
|
|
import sys
|
|
import threading
|
|
|
|
from jarvis.memory.history import HistoryContext
|
|
from voice_worker import brain as brain_mod
|
|
|
|
cfg = tmp_path / "config.json"
|
|
cfg.write_text(json.dumps({"db_path": str(tmp_path / "jarvis.db")}))
|
|
monkeypatch.setenv("JARVIS_CONFIG_PATH", str(cfg))
|
|
|
|
def fake_engine(db, cfg, tts, text, mem, language=None, history=None, legacy_memory=True):
|
|
print(f"answer text {text}")
|
|
print(f"debug {text}", file=sys.stderr)
|
|
# Tools print from helper threads too (the web fetch pool).
|
|
helper = threading.Thread(target=lambda: print(f"helper {text}", file=sys.stderr))
|
|
helper.start()
|
|
helper.join()
|
|
return f"answer to {text}"
|
|
|
|
import jarvis.reply.engine as engine
|
|
monkeypatch.setattr(engine, "run_reply_engine", fake_engine)
|
|
b = brain_mod.JarvisBrain()
|
|
out = []
|
|
t = threading.Thread(target=lambda: out.append(b.reply("100", "secret question", HistoryContext())))
|
|
t.start()
|
|
t.join()
|
|
print("other output stays")
|
|
captured = capfd.readouterr()
|
|
assert out == ["answer to secret question"]
|
|
assert "secret question" not in captured.out + captured.err
|
|
assert "other output stays" in captured.out
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_the_brain_turns_off_the_single_user_memory(tmp_path, monkeypatch):
|
|
pytest.importorskip("mcp") # the real brain needs the jarvis dependencies (--brain)
|
|
from jarvis.memory.history import HistoryContext
|
|
from voice_worker import brain as brain_mod
|
|
|
|
cfg = tmp_path / "config.json"
|
|
cfg.write_text(json.dumps({"db_path": str(tmp_path / "jarvis.db")}))
|
|
monkeypatch.setenv("JARVIS_CONFIG_PATH", str(cfg))
|
|
seen = {}
|
|
|
|
def fake_engine(db, cfg, tts, text, mem, language=None, history=None, legacy_memory=True):
|
|
seen["legacy_memory"] = legacy_memory
|
|
return "ok"
|
|
|
|
import jarvis.reply.engine as engine
|
|
monkeypatch.setattr(engine, "run_reply_engine", fake_engine)
|
|
brain_mod.JarvisBrain().reply("100", "q", HistoryContext())
|
|
assert seen["legacy_memory"] is False
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_the_brain_answers_in_the_askers_language_even_with_a_v1_english_voice(tmp_path, monkeypatch):
|
|
"""v1's ``tts_engine`` (an English-only Piper voice by default) must not lock v2 replies to
|
|
English: v2 speaks through the worker's own TTS, and chat answers are not spoken at all."""
|
|
pytest.importorskip("mcp") # the real brain needs the jarvis dependencies (--brain)
|
|
from jarvis.memory.history import HistoryContext
|
|
from jarvis.system_prompt import reply_language_directive
|
|
from voice_worker import brain as brain_mod
|
|
|
|
cfg = tmp_path / "config.json"
|
|
cfg.write_text(json.dumps({"db_path": str(tmp_path / "jarvis.db"), "tts_engine": "piper"}))
|
|
monkeypatch.setenv("JARVIS_CONFIG_PATH", str(cfg))
|
|
monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False)
|
|
seen = {}
|
|
|
|
def fake_engine(db, cfg, tts, text, mem, language=None, history=None, legacy_memory=True):
|
|
seen["directive"] = reply_language_directive(None, cfg.tts_engine)
|
|
return "ok"
|
|
|
|
import jarvis.reply.engine as engine
|
|
monkeypatch.setattr(engine, "run_reply_engine", fake_engine)
|
|
brain_mod.JarvisBrain().reply("100", "지금 몇 시야?", HistoryContext())
|
|
assert seen["directive"] is None
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_the_brain_tells_tools_who_is_asking(tmp_path, monkeypatch):
|
|
pytest.importorskip("mcp") # the real brain needs the jarvis dependencies (--brain)
|
|
from jarvis.memory.history import HistoryContext
|
|
from jarvis.reply import turn_state
|
|
from voice_worker import brain as brain_mod
|
|
|
|
cfg = tmp_path / "config.json"
|
|
cfg.write_text(json.dumps({"db_path": str(tmp_path / "jarvis.db")}))
|
|
monkeypatch.setenv("JARVIS_CONFIG_PATH", str(cfg))
|
|
seen = []
|
|
|
|
def fake_engine(db, cfg, tts, text, mem, language=None, history=None, legacy_memory=True):
|
|
seen.append(turn_state.get_user())
|
|
return "ok"
|
|
|
|
import jarvis.reply.engine as engine
|
|
monkeypatch.setattr(engine, "run_reply_engine", fake_engine)
|
|
b = brain_mod.JarvisBrain()
|
|
b.reply("100", "q", HistoryContext())
|
|
b.reply("200", "q", HistoryContext())
|
|
assert seen == ["100", "200"]
|
|
assert turn_state.get_user() is None # nothing left behind on this thread
|
|
|
|
|
|
CHANNELS = {"type": "chat_channels", "bot": "900", "channels": [
|
|
{"id": "c1", "name": "일정", "description": "약속", "read": True, "send": True, "edit": True,
|
|
"react": True, "delete": False, "pin": False}]}
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_the_brain_gets_the_registered_channels_and_their_messages(store):
|
|
seen = {}
|
|
|
|
def answer(user, text):
|
|
chat = brain.chat
|
|
seen["channels"] = chat.channels()
|
|
seen["read"] = chat.read("c1", 5)
|
|
seen["search"] = chat.search(["c1"], "회의", 5)
|
|
seen["turn"] = chat.turn
|
|
return "ok"
|
|
|
|
brain = FakeBrain(answer)
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await ws.send(json.dumps(CHANNELS))
|
|
await send(ws, type="chat_message", channel="c1", message_id="m1", author="200", name="Bob",
|
|
text="회의는 3시", ts=1_800_000_000.0)
|
|
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
|
|
await send(ws, type="reply_delivered", id="r1", ok=True)
|
|
await send(ws, type="reply", id="r2", user="100", text="q", via="chat", speak=False)
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r2")
|
|
|
|
asyncio.run(with_worker(body, store=store, brain=brain))
|
|
assert [c["name"] for c in seen["channels"]] == ["일정"] and seen["channels"][0]["send"] is True
|
|
assert [(m["message_id"], m["author_name"], m["text"]) for m in seen["read"]] == [("m1", "Bob", "회의는 3시")]
|
|
assert [m["text"] for m in seen["search"]] == ["회의는 3시"]
|
|
assert seen["turn"] == 2 # the same person's second reply
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_chat_action_goes_to_the_app_and_its_answer_comes_back(store):
|
|
results = []
|
|
brain = FakeBrain(lambda u, t: results.append(brain.chat.act("send", "c1", text="안녕")) or "done")
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await ws.send(json.dumps(CHANNELS))
|
|
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
|
|
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "chat_action")
|
|
action = seen[-1]
|
|
await send(ws, type="chat_action_result", id=action["id"], ok=True, message_ids=["555"])
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
|
|
return action
|
|
|
|
action = asyncio.run(with_worker(body, store=store, brain=brain))
|
|
assert {k: action[k] for k in ("action", "channel", "text")} == {"action": "send", "channel": "c1", "text": "안녕"}
|
|
assert results == [{"ok": True, "message_ids": ["555"]}]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_chat_action_the_app_never_answers_fails(store):
|
|
results = []
|
|
brain = FakeBrain(lambda u, t: results.append(brain.chat.act("send", "c1", text="x")) or "done")
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await ws.send(json.dumps(CHANNELS))
|
|
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
|
|
|
|
worker = VoiceWorker(TOKEN, LevelTranscriber(), AmplitudeVad, None, history=store, brain=brain,
|
|
chat_timeout_s=0.3)
|
|
|
|
async def run():
|
|
server = await worker.serve("127.0.0.1", 0)
|
|
port = server.sockets[0].getsockname()[1]
|
|
try:
|
|
await asyncio.wait_for(body(f"ws://127.0.0.1:{port}"), 15)
|
|
finally:
|
|
server.close()
|
|
await server.wait_closed()
|
|
await worker.stop()
|
|
|
|
asyncio.run(run())
|
|
assert results and results[0]["ok"] is False and results[0]["error"]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_late_answer_from_an_earlier_connection_is_not_taken_for_a_new_action(store):
|
|
results = []
|
|
brain = FakeBrain(lambda u, t: results.append(brain.chat.act("send", "c1", text="x")) or "done")
|
|
worker = VoiceWorker(TOKEN, LevelTranscriber(), AmplitudeVad, None, history=store, brain=brain,
|
|
chat_timeout_s=0.5)
|
|
|
|
async def ask(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await ws.send(json.dumps(CHANNELS))
|
|
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
|
|
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "chat_action")
|
|
return ws, seen[-1]["id"]
|
|
|
|
async def run():
|
|
server = await worker.serve("127.0.0.1", 0)
|
|
url = f"ws://127.0.0.1:{server.sockets[0].getsockname()[1]}"
|
|
try:
|
|
async with connect(url) as ws: # first connection: asked, never answered, then gone
|
|
await hello(ws)
|
|
await ws.send(json.dumps(CHANNELS))
|
|
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
|
|
old = (await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "chat_action"))[-1]["id"]
|
|
async with connect(url) as ws: # second connection: the old answer arrives late
|
|
await hello(ws)
|
|
await ws.send(json.dumps(CHANNELS))
|
|
await send(ws, type="reply", id="r2", user="100", text="q", via="chat", speak=False)
|
|
new = (await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "chat_action"))[-1]["id"]
|
|
await send(ws, type="chat_action_result", id=old, ok=True, message_ids=["stale"])
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r2")
|
|
return old, new
|
|
finally:
|
|
server.close()
|
|
await server.wait_closed()
|
|
await worker.stop()
|
|
|
|
old, new = asyncio.run(asyncio.wait_for(run(), 15))
|
|
assert old != new
|
|
assert all(r.get("message_ids") != ["stale"] for r in results), results
|
|
assert results[-1]["ok"] is False
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_addressed_and_rank_answer_from_the_brain(store):
|
|
brain = FakeBrain()
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await send(ws, type="addressed", id="q1", user="100", text="jarvis, lights off")
|
|
await send(ws, type="addressed", id="q2", user="100", text="did you see that")
|
|
await send(ws, type="rank", id="k1", candidates=[{"id": "7", "user": "100", "text": "a"},
|
|
{"id": "8", "user": "200", "text": "b"}])
|
|
got = {}
|
|
while len(got) < 3:
|
|
m = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") in
|
|
("addressed_result", "rank_result"))
|
|
got[m[-1]["id"]] = m[-1]
|
|
return got
|
|
|
|
got = asyncio.run(with_worker(body, store=store, brain=brain))
|
|
assert got["q1"] == {"type": "addressed_result", "id": "q1", "addressed": True, "failed": False}
|
|
assert got["q2"]["addressed"] is False and got["q2"]["failed"] is False
|
|
assert got["k1"] == {"type": "rank_result", "id": "k1", "order": ["8", "7"], "failed": False}
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_addressed_and_rank_do_not_wait_behind_a_running_reply(store):
|
|
import threading
|
|
release = threading.Event()
|
|
|
|
def slow(user, text):
|
|
release.wait(10)
|
|
return "late"
|
|
|
|
brain = FakeBrain(slow)
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await send(ws, type="reply", id="r1", user="100", text="q", via="voice", speak=False)
|
|
await send(ws, type="rank", id="k1", candidates=[{"id": "7", "user": "100", "text": "a"},
|
|
{"id": "8", "user": "200", "text": "b"}])
|
|
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "rank_result")
|
|
release.set()
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
|
|
return seen
|
|
|
|
seen = asyncio.run(with_worker(body, store=store, brain=brain))
|
|
assert not any(isinstance(m, dict) and m.get("type") == "reply_text" for m in seen)
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("fails", [lambda *a: None, lambda *a: 1 / 0])
|
|
def test_a_failed_judgement_or_ranking_is_reported(store, fails):
|
|
brain = FakeBrain()
|
|
brain.addressed = fails
|
|
brain.rank = fails
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await send(ws, type="addressed", id="q1", user="100", text="x")
|
|
a = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "addressed_result")
|
|
await send(ws, type="rank", id="k1", candidates=[{"id": "7", "user": "100", "text": "a"}])
|
|
r = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "rank_result")
|
|
return a[-1], r[-1]
|
|
|
|
a, r = asyncio.run(with_worker(body, store=store, brain=brain))
|
|
assert a == {"type": "addressed_result", "id": "q1", "addressed": False, "failed": True}
|
|
assert r == {"type": "rank_result", "id": "k1", "order": [], "failed": True}
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_reply_cancelled_while_the_brain_thinks_is_not_spoken(store):
|
|
import threading
|
|
release = threading.Event()
|
|
|
|
def slow(user, text):
|
|
release.wait(10)
|
|
return f"answer for {text}"
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await send(ws, type="voice_join", channel="vc1")
|
|
await send(ws, type="reply", id="r1", user="100", text="q", via="voice", speak=True)
|
|
await send(ws, type="tts_cancel", id="r1")
|
|
await send(ws, type="reply", id="r2", user="100", text="next", via="voice", speak=True)
|
|
await asyncio.sleep(0.1)
|
|
release.set()
|
|
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done"
|
|
and m["id"] == "r2")
|
|
|
|
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain(slow), tts=FakeTts()))
|
|
frames = {decode_tts_frame(m)[0] for m in seen if isinstance(m, bytes)}
|
|
done = {m["id"]: m for m in seen if isinstance(m, dict) and m.get("type") == "tts_done"}
|
|
assert frames == {"r2"}
|
|
assert done["r1"]["cancelled"] and not done["r2"]["cancelled"]
|
|
replies = [m["id"] for m in seen if isinstance(m, dict) and m.get("type") == "reply_text"]
|
|
assert replies == ["r1", "r2"], "the text still comes back; only the speech is skipped"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_the_brains_judgements_use_its_model_and_map_back_to_ids(monkeypatch):
|
|
import voice_worker.judges as judges
|
|
from voice_worker.brain import JarvisBrain
|
|
|
|
asked = []
|
|
|
|
def fake_llm(base_url, model, system, user, **kw):
|
|
asked.append((base_url, model, system, user))
|
|
return "2, 1" if "<requests>" in user else "Yes."
|
|
|
|
monkeypatch.setattr(judges, "call_llm_direct", fake_llm)
|
|
brain = JarvisBrain.__new__(JarvisBrain)
|
|
brain.cfg = type("Cfg", (), {"ollama_base_url": "http://s/v1", "ollama_chat_model": "m"})()
|
|
assert brain.addressed("100", "불 꺼 줘", ["자비스"]) is True
|
|
assert brain.rank([{"id": "7", "user": "1", "text": "a"}, {"id": "8", "user": "2", "text": "b"}]) == ["8", "7"]
|
|
assert asked[0][:2] == ("http://s/v1", "m") and "자비스" in asked[0][2]
|
|
assert asked[0][3] == "<utterance>불 꺼 줘</utterance>", "the heard text is fenced as data"
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("raw,expected", [("no", False), ("NO.", False), ("maybe", None), (None, None)])
|
|
def test_an_unclear_judgement_is_none(monkeypatch, raw, expected):
|
|
import voice_worker.judges as judges
|
|
monkeypatch.setattr(judges, "call_llm_direct", lambda *a, **k: raw)
|
|
assert judges.is_addressed("u", "m", [], "x") is expected
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_ranking_keeps_left_out_requests_in_order_and_fails_without_numbers(monkeypatch):
|
|
import voice_worker.judges as judges
|
|
monkeypatch.setattr(judges, "call_llm_direct", lambda *a, **k: "3 then 9")
|
|
assert judges.rank_requests("u", "m", ["a", "b", "c"]) == [2, 0, 1]
|
|
monkeypatch.setattr(judges, "call_llm_direct", lambda *a, **k: "none of them")
|
|
assert judges.rank_requests("u", "m", ["a", "b"]) is None
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_voice_answer_does_not_wait_behind_a_chat_answer(store):
|
|
import threading
|
|
release = threading.Event()
|
|
running = []
|
|
|
|
def answer(user, text):
|
|
running.append(text)
|
|
if text == "chat question":
|
|
release.wait(10)
|
|
return f"answer for {text}"
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await send(ws, type="reply", id="c1", user="100", text="chat question", via="chat", speak=False)
|
|
await asyncio.sleep(0.1)
|
|
await send(ws, type="reply", id="v1", user="200", text="voice question", via="voice", speak=False)
|
|
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "v1")
|
|
release.set()
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "c1")
|
|
return seen
|
|
|
|
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain(answer)))
|
|
assert seen[-1]["text"] == "answer for voice question"
|
|
assert not any(isinstance(m, dict) and m.get("id") == "c1" for m in seen), "the chat answer was still running"
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("via", ["voice", "chat"])
|
|
def test_answers_of_one_kind_are_written_one_at_a_time(store, via):
|
|
import threading
|
|
active, peak = [0], [0]
|
|
lock = threading.Lock()
|
|
|
|
def answer(user, text):
|
|
with lock:
|
|
active[0] += 1
|
|
peak[0] = max(peak[0], active[0])
|
|
import time as _t
|
|
_t.sleep(0.05)
|
|
with lock:
|
|
active[0] -= 1
|
|
return "ok"
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
for i in range(3):
|
|
await send(ws, type="reply", id=f"r{i}", user="100", text="q", via=via, speak=False)
|
|
for i in range(3):
|
|
await recv_until(ws, lambda m, i=i: isinstance(m, dict) and m.get("id") == f"r{i}")
|
|
|
|
asyncio.run(with_worker(body, store=store, brain=FakeBrain(answer)))
|
|
assert peak[0] == 1
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_one_persons_voice_and_chat_answers_are_written_one_at_a_time(store):
|
|
# Their dialogue memory and delete confirmations ("the very next message") follow the order
|
|
# they asked, so a person's second question starts only after their first is answered.
|
|
import threading
|
|
release = threading.Event()
|
|
running = []
|
|
|
|
def answer(user, text):
|
|
running.append(text)
|
|
if text == "chat question":
|
|
release.wait(10)
|
|
return f"answer for {text}"
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await send(ws, type="reply", id="c1", user="100", text="chat question", via="chat", speak=False)
|
|
await asyncio.sleep(0.1)
|
|
await send(ws, type="reply", id="v1", user="100", text="voice question", via="voice", speak=False)
|
|
await asyncio.sleep(0.3)
|
|
started = list(running)
|
|
release.set()
|
|
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "v1")
|
|
return started, [m["id"] for m in seen if isinstance(m, dict) and m.get("type") == "reply_text"]
|
|
|
|
started, order = asyncio.run(with_worker(body, store=store, brain=FakeBrain(answer)))
|
|
assert started == ["chat question"], "the voice question started before the same person's chat answer"
|
|
assert order == ["c1", "v1"]
|
|
|
|
|
|
DELETABLE = {"type": "chat_channels", "bot": "900", "channels": [
|
|
{"id": "c1", "name": "일정", "description": "약속", "read": True, "send": True, "edit": True,
|
|
"react": True, "delete": True, "pin": False}]}
|
|
|
|
|
|
def delete_tool_brain(release, question="지울까요?"):
|
|
"""A brain whose reply runs the real chatChannels tool: "지워" asks to delete message m1 (and waits
|
|
for ``release`` before answering), "응, 지워" confirms. The consent check always agrees."""
|
|
from unittest.mock import patch
|
|
|
|
from jarvis.reply import turn_state
|
|
from jarvis.tools.builtin.chat_channels import ChatChannelsTool
|
|
|
|
class Cfg:
|
|
ollama_base_url = "http://x"
|
|
ollama_chat_model = "m"
|
|
|
|
results = []
|
|
|
|
def answer(user, text):
|
|
turn_state.reset()
|
|
turn_state.set_chat(brain.chat)
|
|
args = {"action": "delete", "channel": "c1", "message_id": "m1", "confirm": text == "응, 지워"}
|
|
try:
|
|
with patch("jarvis.tools.builtin.chat_channels.call_llm_direct", return_value="YES"):
|
|
res = ChatChannelsTool().execute(db=None, cfg=Cfg(), tool_args=args, system_prompt="",
|
|
original_prompt=text, redacted_text=text, max_retries=0,
|
|
user_print=lambda m: None)
|
|
finally:
|
|
turn_state.reset()
|
|
results.append((text, res.success))
|
|
if text == "지워":
|
|
release.wait(10)
|
|
return question if text == "지워" else "done"
|
|
|
|
brain = FakeBrain(answer)
|
|
return brain, results
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_yes_sent_before_the_delete_question_was_answered_does_not_delete(store):
|
|
import threading
|
|
release = threading.Event()
|
|
brain, results = delete_tool_brain(release)
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await ws.send(json.dumps(DELETABLE))
|
|
await send(ws, type="reply", id="r1", user="100", text="지워", via="chat", speak=False)
|
|
await asyncio.sleep(0.2)
|
|
# Typed before the question reached them: it cannot be their answer to it.
|
|
await send(ws, type="reply", id="r2", user="100", text="응, 지워", via="voice", speak=False)
|
|
await asyncio.sleep(0.1)
|
|
release.set()
|
|
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r2")
|
|
|
|
seen = asyncio.run(with_worker(body, store=store, brain=brain))
|
|
assert not any(isinstance(m, dict) and m.get("type") == "chat_action" for m in seen)
|
|
assert results == [("지워", True), ("응, 지워", False)]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_yes_after_the_delete_question_deletes(store):
|
|
import threading
|
|
release = threading.Event()
|
|
release.set()
|
|
brain, results = delete_tool_brain(release)
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await ws.send(json.dumps(DELETABLE))
|
|
await send(ws, type="reply", id="r1", user="100", text="지워", via="chat", speak=False)
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r1")
|
|
await send(ws, type="reply_delivered", id="r1", ok=True) # the app posted the question
|
|
await send(ws, type="reply", id="r2", user="100", text="응, 지워", via="voice", speak=False)
|
|
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "chat_action")
|
|
await send(ws, type="chat_action_result", id=seen[-1]["id"], ok=True)
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r2")
|
|
return seen[-1]
|
|
|
|
action = asyncio.run(with_worker(body, store=store, brain=brain))
|
|
assert (action["action"], action["channel"], action["message_id"]) == ("delete", "c1", "m1")
|
|
assert results == [("지워", True), ("응, 지워", True)]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_yes_after_a_delete_question_they_never_got_does_not_delete(store):
|
|
import threading
|
|
release = threading.Event()
|
|
release.set()
|
|
brain, results = delete_tool_brain(release, question="") # the answer failed: nothing was asked
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await ws.send(json.dumps(DELETABLE))
|
|
await send(ws, type="reply", id="r1", user="100", text="지워", via="chat", speak=False)
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r1")
|
|
await send(ws, type="reply", id="r2", user="100", text="응, 지워", via="voice", speak=False)
|
|
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r2")
|
|
|
|
seen = asyncio.run(with_worker(body, store=store, brain=brain))
|
|
assert not any(isinstance(m, dict) and m.get("type") == "chat_action" for m in seen)
|
|
assert results == [("지워", True), ("응, 지워", False)]
|
|
|
|
|
|
def run_delete_then_yes(store, ask, after, question="지울까요?", tts=None, **kw):
|
|
"""The person asks to delete (``ask``: the reply message fields), ``after(ws)`` plays what happened
|
|
to the question, then they say "응, 지워" by voice. The chat actions that went out."""
|
|
import threading
|
|
release = threading.Event()
|
|
release.set()
|
|
brain, results = delete_tool_brain(release, question=question)
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await ws.send(json.dumps(DELETABLE))
|
|
await send(ws, type="reply", id="r1", user="100", text="지워", **ask)
|
|
await after(ws)
|
|
await send(ws, type="reply", id="r2", user="100", text="응, 지워", via="voice", speak=False)
|
|
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r2")
|
|
return [m for m in seen if isinstance(m, dict) and m.get("type") == "chat_action"]
|
|
|
|
actions = asyncio.run(with_worker(body, store=store, brain=brain, tts=tts, **kw))
|
|
return actions, results
|
|
|
|
|
|
def until(pred):
|
|
async def after(ws):
|
|
await recv_until(ws, pred)
|
|
return after
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_yes_after_a_question_that_could_not_be_spoken_does_not_delete(store):
|
|
# No TTS engine: the spoken question never reached them.
|
|
actions, results = run_delete_then_yes(
|
|
store, dict(via="voice", speak=True),
|
|
until(lambda m: isinstance(m, dict) and m.get("code") == "tts_unavailable"))
|
|
assert actions == [] and results == [("지워", True), ("응, 지워", False)]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_yes_after_a_question_whose_speech_failed_does_not_delete(store):
|
|
actions, results = run_delete_then_yes(
|
|
store, dict(via="voice", speak=True),
|
|
until(lambda m: isinstance(m, dict) and m.get("type") == "tts_done" and m["failed"]),
|
|
question="FAIL 지울까요?", tts=FakeTts())
|
|
assert actions == [] and results == [("지워", True), ("응, 지워", False)]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_yes_after_a_spoken_question_deletes(store):
|
|
import threading
|
|
release = threading.Event()
|
|
release.set()
|
|
brain, results = delete_tool_brain(release)
|
|
|
|
async def body(url):
|
|
async with connect(url) as ws:
|
|
await hello(ws)
|
|
await ws.send(json.dumps(DELETABLE))
|
|
await send(ws, type="reply", id="r1", user="100", text="지워", via="voice", speak=True)
|
|
done = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
|
|
assert not done[-1]["failed"] and not done[-1]["cancelled"]
|
|
await send(ws, type="reply", id="r2", user="100", text="응, 지워", via="voice", speak=False)
|
|
seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "chat_action")
|
|
await send(ws, type="chat_action_result", id=seen[-1]["id"], ok=True)
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r2")
|
|
return seen[-1]
|
|
|
|
action = asyncio.run(with_worker(body, store=store, brain=brain, tts=FakeTts()))
|
|
assert action["action"] == "delete" and results == [("지워", True), ("응, 지워", True)]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_yes_after_a_chat_question_the_app_could_not_post_does_not_delete(store):
|
|
async def not_posted(ws):
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r1")
|
|
await send(ws, type="reply_delivered", id="r1", ok=False)
|
|
|
|
actions, results = run_delete_then_yes(store, dict(via="chat", speak=False), not_posted)
|
|
assert actions == [] and results == [("지워", True), ("응, 지워", False)]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_chat_question_the_app_never_reports_counts_as_not_posted(store):
|
|
async def no_report(ws):
|
|
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r1")
|
|
await asyncio.sleep(0.3)
|
|
|
|
actions, results = run_delete_then_yes(store, dict(via="chat", speak=False), no_report,
|
|
delivery_timeout_s=0.1)
|
|
assert actions == [] and results == [("지워", True), ("응, 지워", False)]
|