Files
javis_bot/src/voice_worker/server.py
javis-bot fe4e296361 feat(chat): chatChannels tool for the registered Discord chat channels
- Tool (chat_channels.spec.md): list, read, search, send, edit, react,
  delete, pin, unpin. Offered only when the v2 brain gives the turn a chat
  backend, and then on every turn. Channel by ID, name, part of a name or a
  name inside the text; a message ID finds its channel; planner forms of the
  action ("send_message") work. Read and search results are fenced as
  untrusted data. delete only reports until called again with confirm=true
  in a later turn of the same person.
- Worker: chat_channels from the app, ChatBridge reads/searches the history
  store and sends chat_action, waiting for chat_action_result (15 s timeout,
  failed on session close); per-person reply turns and pending deletes.
- App: channel switches edit/react (on) and delete/pin (off); chat_channels
  sent on worker ready and after login; chat_action runs send, edit (own
  messages only), react, delete, pin, unpin and answers with ids or the error.
- Engine: chatChannels left out of the catalogue without a backend; the
  "router returned everything" check now tolerates that.
- live_discord: edit own, refuse editing the speaker's, react, pin (Discord
  refuses in a voice channel's chat, reported as an error), delete, refuse an
  unregistered channel; passed in 109.4 s.
- Eval (qwen2.5:7b, 3 runs): 10 of 12; never obeyed the "delete everything"
  message and never deleted in the same turn.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 14:01:33 +09:00

531 lines
24 KiB
Python

"""Local WebSocket server: one session per connection (protocol in voice_worker.spec.md)."""
from __future__ import annotations
import asyncio
import hmac
import ipaddress
import itertools
import json
import time
from dataclasses import dataclass
from typing import Callable
import numpy as np
from .log import debug_log
from .audio import InputConverter, loudness_equalise
from .brain import Brain, MemorySettings
from .protocol import KIND_AUDIO, ProtocolError, decode_audio_frame, encode_tts_frame
from .scheduler import DecodeOptions, SttResult, SttScheduler, Transcriber
from .segmenter import SR, Final, Partial, Segmenter, SpeechStart, VadStream
from .settings import SttSettings
from .tts import TtsEngine, TtsRequest, TtsService
CLOSE_BAD_HELLO = 4400
CLOSE_BAD_TOKEN = 4401
MAX_SPEAKERS = 32
def is_loopback(host: str) -> bool:
if host == "localhost":
return True
try:
return ipaddress.ip_address(host).is_loopback
except ValueError:
return False
class VoiceWorker:
def __init__(self, token: str, transcriber: Transcriber | None, vad_factory: Callable[[], VadStream],
tts_engine: TtsEngine | None = None, hello_timeout_s: float = 5.0, tick_s: float = 0.05,
history=None, brain: Brain | None = None, chat_timeout_s: float = 15.0):
if not token:
raise ValueError("a connection token is required")
self.token = token
self.transcriber = transcriber
self.vad_factory = vad_factory
self.tts_engine = tts_engine
self.hello_timeout_s = hello_timeout_s
self.tick_s = tick_s
self.history = history # jarvis.memory.history.HistoryStore or None
self.brain = brain
self.brain_lock = asyncio.Lock() # one reply at a time
self.chat_timeout_s = chat_timeout_s
# Per person: how many replies they had (a delete is confirmed only in a later one) and the
# deletes they were asked to confirm (chat_channels.spec.md).
self.reply_turns: dict[str, int] = {}
self.pending_deletes: dict[str, dict] = {}
self.scheduler: SttScheduler | None = None
self.tts: TtsService | None = None
self._ids = itertools.count(1)
async def start(self) -> None:
if self.history is not None:
# A killed worker never closed its sessions' presence; nobody is heard until the app
# reports who is there again.
n = self.history.close_open_presence(time.time())
if n:
debug_log(f"closed {n} presence intervals left open by a previous run", "voice_worker")
if self.transcriber is not None:
self.scheduler = SttScheduler(self.transcriber)
self.scheduler.start()
if self.tts_engine is not None:
self.tts = TtsService(self.tts_engine)
self.tts.start()
async def stop(self) -> None:
if self.scheduler:
await self.scheduler.stop()
if self.tts:
await self.tts.stop()
async def serve(self, host: str, port: int):
"""Start listening; returns the websockets server. Loopback addresses only."""
if not is_loopback(host):
raise ValueError("the voice worker only listens on a loopback address")
from websockets.asyncio.server import serve
await self.start()
return await serve(self.handle, host, port, max_size=2 ** 20)
async def handle(self, ws) -> None:
try:
first = await asyncio.wait_for(ws.recv(), self.hello_timeout_s)
hello = json.loads(first) if isinstance(first, str) else None
except (asyncio.TimeoutError, ValueError):
hello = None
except Exception:
return
if not isinstance(hello, dict) or hello.get("type") != "hello":
debug_log("connection refused: no hello", "voice_worker")
await ws.close(CLOSE_BAD_HELLO, "hello expected")
return
token = hello.get("token")
if not isinstance(token, str) or not hmac.compare_digest(token.encode(), self.token.encode()):
debug_log("connection refused: bad token", "voice_worker")
await ws.close(CLOSE_BAD_TOKEN, "bad token")
return
try:
session = Session(self, ws, next(self._ids), hello)
except ValueError as e:
await ws.close(CLOSE_BAD_HELLO, str(e)[:100])
return
await session.run()
@dataclass
class _Speaker:
converter: InputConverter
segmenter: Segmenter
def _format(value, default: dict, rates: tuple[int, ...] | None) -> dict:
if value is None:
return dict(default)
if not isinstance(value, dict):
raise ValueError("format must be an object")
rate, channels = value.get("rate", default["rate"]), value.get("channels", default["channels"])
ok_rate = isinstance(rate, int) and (rate in rates if rates else 8000 <= rate <= 96000)
if not ok_rate or channels not in (1, 2) or isinstance(channels, bool):
raise ValueError("unsupported audio format")
return {"rate": rate, "channels": channels}
class Session:
def __init__(self, worker: VoiceWorker, ws, sid: int, hello: dict):
self.worker = worker
self.ws = ws
self.sid = sid
self.audio = _format(hello.get("audio"), {"rate": 48000, "channels": 2}, (16000, 48000))
self.tts_format = _format(hello.get("tts"), {"rate": 48000, "channels": 2}, None)
self.settings = SttSettings()
stt = hello.get("stt")
if stt is not None:
if not isinstance(stt, dict):
raise ValueError("stt must be an object")
self.settings.update(stt)
self.memory = MemorySettings()
memory = hello.get("memory")
if memory is not None:
if not isinstance(memory, dict):
raise ValueError("memory must be an object")
self.memory.update(memory)
self.voice_channel: str | None = None
self.chat_channels: list[dict] = []
self.chat_waiting: dict[str, asyncio.Future] = {}
self._chat_ids = itertools.count(1)
self.names: dict[str, str] = {}
self.speakers: dict[str, _Speaker] = {}
self.tts_requests: dict[str, TtsRequest] = {}
self.out: asyncio.Queue = asyncio.Queue()
self.closed = False
# --- outbound -------------------------------------------------------------------------------
def send(self, msg) -> None:
if not self.closed:
self.out.put_nowait(msg if isinstance(msg, bytes) else json.dumps(msg, ensure_ascii=False))
def error(self, code: str, message: str, **extra) -> None:
self.send({"type": "error", "code": code, "message": message, **extra})
async def _writer(self) -> None:
while True:
await self.ws.send(await self.out.get())
# --- lifecycle ------------------------------------------------------------------------------
async def run(self) -> None:
w = self.worker
debug_log(f"session {self.sid} connected audio={self.audio} tts={self.tts_format}", "voice_worker")
self.send({"type": "hello_ok", "stt_ready": w.scheduler is not None, "tts_ready": w.tts is not None,
"audio": self.audio, "tts": self.tts_format, "stt": self.settings.as_dict()})
writer = asyncio.create_task(self._writer())
ticker = asyncio.create_task(self._ticker())
try:
async for frame in self.ws:
if isinstance(frame, bytes):
self._on_binary(frame)
else:
self._on_text(frame)
except Exception as e: # connection dropped
debug_log(f"session {self.sid} receive ended: {type(e).__name__}", "voice_worker")
finally:
self.closed = True
self._leave_voice()
for fut in self.chat_waiting.values():
if not fut.done():
fut.set_result({"ok": False, "error": "the app disconnected"})
ticker.cancel()
writer.cancel()
if w.scheduler:
w.scheduler.drop_partials(lambda key: key[0] == self.sid)
if w.tts:
for req in list(self.tts_requests.values()):
w.tts.cancel(req)
debug_log(f"session {self.sid} closed", "voice_worker")
async def _ticker(self) -> None:
while True:
await asyncio.sleep(self.worker.tick_s)
now = time.monotonic()
for speaker, st in list(self.speakers.items()):
self._handle_events(speaker, st, st.segmenter.tick(now))
# --- STT ------------------------------------------------------------------------------------
def _on_binary(self, frame: bytes) -> None:
if not frame or frame[0] != KIND_AUDIO:
self.error("bad_frame", "unknown binary frame kind")
return
try:
speaker, pcm = decode_audio_frame(frame)
except ProtocolError as e:
self.error("bad_frame", str(e))
return
st = self.speakers.get(speaker)
if st is None:
if len(self.speakers) >= MAX_SPEAKERS:
self.error("bad_frame", f"more than {MAX_SPEAKERS} speakers")
return
st = _Speaker(InputConverter(self.audio["rate"], self.audio["channels"]),
Segmenter(self.worker.vad_factory(), self.settings))
self.speakers[speaker] = st
samples = st.converter.push(pcm)
self._handle_events(speaker, st, st.segmenter.push(samples, time.monotonic()))
def _opts(self, beam: int) -> DecodeOptions:
return DecodeOptions(beam=beam, language=self.settings.language, hotwords=self.settings.hotwords)
def _handle_events(self, speaker: str, st: _Speaker, events: list) -> None:
sched = self.worker.scheduler
for ev in events:
if isinstance(ev, SpeechStart):
self.send({"type": "speech_start", "speaker": speaker, "utt": ev.utt})
elif isinstance(ev, Partial):
if sched and self.settings.partials:
sched.submit_partial((self.sid, speaker), loudness_equalise(ev.audio, ev.voiced), self._opts(1),
self._partial_cb(speaker, st, ev.utt))
elif isinstance(ev, Final):
debug_log(f"utterance end speaker={speaker} utt={ev.utt} reason={ev.reason} "
f"ms={len(ev.audio) * 1000 // SR}", "voice_worker")
# Where, when and by whom it was said, fixed now: decoding can take a while, and a
# move or a late joiner meanwhile must not change who heard it.
said = (self.voice_channel, self.names.get(speaker, speaker),
time.time() - (time.monotonic() - ev.decided_at))
if sched is None:
self._send_final(speaker, ev, SttResult("", 0, "stt unavailable"), said)
else:
sched.submit_final((self.sid, speaker), loudness_equalise(ev.audio, ev.voiced),
self._opts(self.settings.final_beam),
lambda res, ev=ev, said=said: self._send_final(speaker, ev, res, said))
def _partial_cb(self, speaker: str, st: _Speaker, utt: int):
def cb(res: SttResult) -> None:
# Drop it when the utterance already ended or captions were turned off meanwhile.
if st.segmenter.open_utt == utt and self.settings.partials and not res.error:
self.send({"type": "partial", "speaker": speaker, "utt": utt, "text": res.text})
return cb
def _send_final(self, speaker: str, ev: Final, res: SttResult, said: tuple) -> None:
channel, name, ts = said
store = self.worker.history
if store is not None and channel and res.text.strip():
store.add_voice(channel, speaker, name, res.text, ts)
self.send({"type": "final", "speaker": speaker, "utt": ev.utt, "text": res.text,
"end_reason": ev.reason, "audio_ms": len(ev.audio) * 1000 // SR,
"wait_ms": int((time.monotonic() - ev.decided_at) * 1000), "stt_ms": res.stt_ms})
# --- text messages --------------------------------------------------------------------------
def _on_text(self, raw: str) -> None:
try:
msg = json.loads(raw)
except ValueError:
self.error("bad_message", "not JSON")
return
if not isinstance(msg, dict):
self.error("bad_message", "not an object")
return
kind = msg.get("type")
handler = {"speaking_end": self._on_speaking_end, "speaker_leave": self._on_speaker_leave,
"config": self._on_config, "tts": self._on_tts, "tts_cancel": self._on_tts_cancel,
"voice_join": self._on_voice_join, "voice_leave": lambda m: self._leave_voice(),
"member_join": self._on_member_join, "member_leave": self._on_member_leave,
"chat_message": self._on_chat_message, "chat_edit": self._on_chat_edit,
"chat_delete": self._on_chat_delete, "reply": self._on_reply,
"chat_channels": self._on_chat_channels, "chat_action_result": self._on_chat_action_result,
"memory_config": self._on_memory_config}.get(kind)
if handler is None:
self.error("bad_message", f"unknown type: {kind!r}")
return
handler(msg)
def _on_speaking_end(self, msg: dict) -> None:
st = self.speakers.get(msg.get("speaker"))
if st is not None:
self._handle_events(msg["speaker"], st, st.segmenter.speaking_end(time.monotonic()))
def _on_speaker_leave(self, msg: dict) -> None:
# The speaker left the voice channel: finish any open utterance, then free their place.
speaker = msg.get("speaker")
st = self.speakers.pop(speaker, None) if isinstance(speaker, str) else None
if st is not None:
self._handle_events(speaker, st, st.segmenter.speaking_end(time.monotonic()))
if self.worker.scheduler:
self.worker.scheduler.drop_partials(lambda key: key == (self.sid, speaker))
debug_log(f"speaker {speaker} left session {self.sid}", "voice_worker")
def _on_config(self, msg: dict) -> None:
changes = {k: v for k, v in msg.items() if k != "type"}
try:
self.settings.update(changes)
except ValueError as e:
self.error("bad_message", str(e))
return
if not self.settings.partials and self.worker.scheduler:
self.worker.scheduler.drop_partials(lambda key: key[0] == self.sid)
debug_log(f"session {self.sid} settings changed: {sorted(changes)}", "voice_worker")
def _on_tts(self, msg: dict) -> None:
rid, text, more = msg.get("id"), msg.get("text", ""), msg.get("more", False)
if not isinstance(rid, str) or not 0 < len(rid.encode()) < 256 or not isinstance(text, str) \
or not isinstance(more, bool):
self.error("bad_message", "tts needs a string id and text, and a boolean more")
return
tts = self.worker.tts
if tts is None:
self.error("tts_unavailable", "no TTS engine is loaded", id=rid)
return
req = self.tts_requests.get(rid)
if req is None:
req = TtsRequest(rid, self.tts_format["rate"], self._tts_chunk_cb(rid), self._tts_done,
self._tts_error)
self.tts_requests[rid] = req
elif req.closed:
self.error("bad_message", "this tts id is already closed", id=rid)
return
tts.feed(req, text, more)
def _on_tts_cancel(self, msg: dict) -> None:
req = self.tts_requests.get(msg.get("id"))
if req is not None and self.worker.tts:
self.worker.tts.cancel(req)
# --- history and brain ----------------------------------------------------------------------
def _strings(self, msg: dict, *keys: str) -> tuple | None:
values = tuple(msg.get(k) for k in keys)
if not all(isinstance(v, str) and v for v in values):
self.error("bad_message", f"{msg.get('type')} needs non-empty strings: {', '.join(keys)}",
**({"id": msg["id"]} if isinstance(msg.get("id"), str) else {}))
return None
return values
def _leave_voice(self) -> None:
if self.voice_channel and self.worker.history is not None:
self.worker.history.presence_close_channel(self.voice_channel, time.time())
if self.voice_channel:
debug_log(f"session {self.sid} left voice channel {self.voice_channel}", "voice_worker")
self.voice_channel = None
def _on_voice_join(self, msg: dict) -> None:
got = self._strings(msg, "channel")
if got is None:
return
self._leave_voice()
self.voice_channel = got[0]
debug_log(f"session {self.sid} in voice channel {self.voice_channel}", "voice_worker")
def _on_member_join(self, msg: dict) -> None:
got = self._strings(msg, "user", "name")
if got is None:
return
user, name = got
self.names[user] = name
if self.voice_channel and self.worker.history is not None:
self.worker.history.presence_join(user, self.voice_channel, time.time())
def _on_member_leave(self, msg: dict) -> None:
got = self._strings(msg, "user")
if got is not None and self.voice_channel and self.worker.history is not None:
self.worker.history.presence_leave(got[0], self.voice_channel, time.time())
def _on_chat_message(self, msg: dict) -> None:
got = self._strings(msg, "channel", "message_id", "author", "name")
ts, to, text = msg.get("ts"), msg.get("to"), msg.get("text")
if got is None:
return
if not isinstance(text, str) or isinstance(ts, bool) or not isinstance(ts, (int, float)) \
or (to is not None and not isinstance(to, str)):
self.error("bad_message", "chat_message needs text, a numeric ts and an optional string to")
return
if self.worker.history is not None:
channel, message_id, author, name = got
self.worker.history.add_chat(channel, message_id, author, name, text, float(ts), to_id=to)
def _on_chat_edit(self, msg: dict) -> None:
got = self._strings(msg, "channel", "message_id")
if got is None:
return
if not isinstance(msg.get("text"), str):
self.error("bad_message", "chat_edit needs text")
return
if self.worker.history is not None:
self.worker.history.edit_chat(got[0], got[1], msg["text"])
def _on_chat_delete(self, msg: dict) -> None:
got = self._strings(msg, "channel", "message_id")
if got is not None and self.worker.history is not None:
self.worker.history.delete_chat(*got)
_CHAT_FLAGS = ("read", "send", "edit", "react", "delete", "pin")
def _on_chat_channels(self, msg: dict) -> None:
channels = msg.get("channels")
ok = isinstance(channels, list) and all(
isinstance(c, dict) and all(isinstance(c.get(k), str) for k in ("id", "name", "description"))
and c["id"] and all(isinstance(c.get(f), bool) for f in self._CHAT_FLAGS) for c in channels)
if not ok:
self.error("bad_message", "chat_channels needs channels with id, name, description and "
"boolean read, send, edit, react, delete, pin")
return
self.chat_channels = [{k: c[k] for k in ("id", "name", "description", *self._CHAT_FLAGS)} for c in channels]
debug_log(f"session {self.sid} chat channels: {len(self.chat_channels)}", "voice_worker")
async def chat_action(self, action: str, channel: str, args: dict) -> dict:
"""Ask the app to do a chat action and wait for its answer (or a timeout)."""
aid = f"a{next(self._chat_ids)}"
fut = asyncio.get_running_loop().create_future()
self.chat_waiting[aid] = fut
self.send({"type": "chat_action", "id": aid, "action": action, "channel": channel,
**{k: v for k, v in args.items() if v is not None}})
debug_log(f"chat action {aid} {action} channel={channel}", "voice_worker")
try:
return await asyncio.wait_for(fut, self.worker.chat_timeout_s)
except asyncio.TimeoutError:
return {"ok": False, "error": "the app did not answer in time"}
finally:
self.chat_waiting.pop(aid, None)
def _on_chat_action_result(self, msg: dict) -> None:
fut = self.chat_waiting.get(msg.get("id"))
if fut is None or fut.done():
return
ids = msg.get("message_ids") or []
result = {"ok": msg.get("ok") is True}
if result["ok"]:
result["message_ids"] = [str(i) for i in ids if isinstance(i, (str, int))]
else:
result["error"] = str(msg.get("error") or "the app could not do it")
fut.set_result(result)
def _on_memory_config(self, msg: dict) -> None:
changes = {k: v for k, v in msg.items() if k != "type"}
try:
self.memory.update(changes)
except ValueError as e:
self.error("bad_message", str(e))
return
debug_log(f"session {self.sid} memory settings changed: {sorted(changes)}", "voice_worker")
def _on_reply(self, msg: dict) -> None:
got = self._strings(msg, "id", "user", "text")
if got is None:
return
rid, user, text = got
via, speak = msg.get("via"), msg.get("speak", False)
if via not in ("voice", "chat") or not isinstance(speak, bool) or len(rid.encode()) > 255:
self.error("bad_message", "reply needs via voice|chat, a boolean speak and an id under 256 bytes",
id=rid)
return
if self.worker.brain is None:
self.error("brain_unavailable", "the worker was started without --brain", id=rid)
return
asyncio.get_running_loop().create_task(self._reply(rid, user, text, via, speak))
async def _reply(self, rid: str, user: str, text: str, via: str, speak: bool) -> None:
w = self.worker
channel = self.voice_channel
async with w.brain_lock:
if self.closed:
return
now = time.time()
if w.history is not None:
m = self.memory
history = await asyncio.to_thread(
w.history.context, user, text, now, m.scope(now), minutes=m.recent_minutes,
max_records=m.recent_records, limit=m.matches, budget_tokens=m.budget_tokens)
else:
from jarvis.memory.history import HistoryContext
history = HistoryContext()
turn = w.reply_turns[user] = w.reply_turns.get(user, 0) + 1
from .chat_bridge import ChatBridge
chat = ChatBridge(self, asyncio.get_running_loop(), turn, w.pending_deletes.setdefault(user, {})) \
if self.chat_channels else None
try:
answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat)
except Exception as e: # the brain must never take the session down
debug_log(f"brain reply {rid} failed: {type(e).__name__}", "voice_worker")
answer = None
answer = (answer or "").strip()
self.send({"type": "reply_text", "id": rid, "text": answer, "failed": not answer})
if not answer:
return
if via == "voice" and channel and w.history is not None:
w.history.add_reply(channel, user, answer, time.time())
if speak:
self._on_tts({"type": "tts", "id": rid, "text": answer, "more": False})
def _tts_chunk_cb(self, rid: str):
def cb(seq: int, pcm: np.ndarray) -> None:
pcm = np.asarray(pcm, dtype="<i2")
if self.tts_format["channels"] == 2:
pcm = np.repeat(pcm, 2)
self.send(encode_tts_frame(rid, seq, pcm.tobytes()))
return cb
def _tts_done(self, req: TtsRequest) -> None:
self.tts_requests.pop(req.id, None)
self.send({"type": "tts_done", "id": req.id, "chunks": req.chunks, "cancelled": req.cancelled,
"failed": req.failed})
def _tts_error(self, req: TtsRequest, message: str) -> None:
self.error("tts_failed", message, id=req.id)