- Tool (chat_channels.spec.md): list, read, search, send, edit, react,
delete, pin, unpin. Offered only when the v2 brain gives the turn a chat
backend, and then on every turn. Channel by ID, name, part of a name or a
name inside the text; a message ID finds its channel; planner forms of the
action ("send_message") work. Read and search results are fenced as
untrusted data. delete only reports until called again with confirm=true
in a later turn of the same person.
- Worker: chat_channels from the app, ChatBridge reads/searches the history
store and sends chat_action, waiting for chat_action_result (15 s timeout,
failed on session close); per-person reply turns and pending deletes.
- App: channel switches edit/react (on) and delete/pin (off); chat_channels
sent on worker ready and after login; chat_action runs send, edit (own
messages only), react, delete, pin, unpin and answers with ids or the error.
- Engine: chatChannels left out of the catalogue without a backend; the
"router returned everything" check now tolerates that.
- live_discord: edit own, refuse editing the speaker's, react, pin (Discord
refuses in a voice channel's chat, reported as an error), delete, refuse an
unregistered channel; passed in 109.4 s.
- Eval (qwen2.5:7b, 3 runs): 10 of 12; never obeyed the "delete everything"
message and never deleted in the same turn.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
531 lines
24 KiB
Python
531 lines
24 KiB
Python
"""Local WebSocket server: one session per connection (protocol in voice_worker.spec.md)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hmac
|
|
import ipaddress
|
|
import itertools
|
|
import json
|
|
import time
|
|
from dataclasses import dataclass
|
|
from typing import Callable
|
|
|
|
import numpy as np
|
|
|
|
from .log import debug_log
|
|
|
|
from .audio import InputConverter, loudness_equalise
|
|
from .brain import Brain, MemorySettings
|
|
from .protocol import KIND_AUDIO, ProtocolError, decode_audio_frame, encode_tts_frame
|
|
from .scheduler import DecodeOptions, SttResult, SttScheduler, Transcriber
|
|
from .segmenter import SR, Final, Partial, Segmenter, SpeechStart, VadStream
|
|
from .settings import SttSettings
|
|
from .tts import TtsEngine, TtsRequest, TtsService
|
|
|
|
CLOSE_BAD_HELLO = 4400
|
|
CLOSE_BAD_TOKEN = 4401
|
|
MAX_SPEAKERS = 32
|
|
|
|
|
|
def is_loopback(host: str) -> bool:
|
|
if host == "localhost":
|
|
return True
|
|
try:
|
|
return ipaddress.ip_address(host).is_loopback
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
class VoiceWorker:
|
|
def __init__(self, token: str, transcriber: Transcriber | None, vad_factory: Callable[[], VadStream],
|
|
tts_engine: TtsEngine | None = None, hello_timeout_s: float = 5.0, tick_s: float = 0.05,
|
|
history=None, brain: Brain | None = None, chat_timeout_s: float = 15.0):
|
|
if not token:
|
|
raise ValueError("a connection token is required")
|
|
self.token = token
|
|
self.transcriber = transcriber
|
|
self.vad_factory = vad_factory
|
|
self.tts_engine = tts_engine
|
|
self.hello_timeout_s = hello_timeout_s
|
|
self.tick_s = tick_s
|
|
self.history = history # jarvis.memory.history.HistoryStore or None
|
|
self.brain = brain
|
|
self.brain_lock = asyncio.Lock() # one reply at a time
|
|
self.chat_timeout_s = chat_timeout_s
|
|
# Per person: how many replies they had (a delete is confirmed only in a later one) and the
|
|
# deletes they were asked to confirm (chat_channels.spec.md).
|
|
self.reply_turns: dict[str, int] = {}
|
|
self.pending_deletes: dict[str, dict] = {}
|
|
self.scheduler: SttScheduler | None = None
|
|
self.tts: TtsService | None = None
|
|
self._ids = itertools.count(1)
|
|
|
|
async def start(self) -> None:
|
|
if self.history is not None:
|
|
# A killed worker never closed its sessions' presence; nobody is heard until the app
|
|
# reports who is there again.
|
|
n = self.history.close_open_presence(time.time())
|
|
if n:
|
|
debug_log(f"closed {n} presence intervals left open by a previous run", "voice_worker")
|
|
if self.transcriber is not None:
|
|
self.scheduler = SttScheduler(self.transcriber)
|
|
self.scheduler.start()
|
|
if self.tts_engine is not None:
|
|
self.tts = TtsService(self.tts_engine)
|
|
self.tts.start()
|
|
|
|
async def stop(self) -> None:
|
|
if self.scheduler:
|
|
await self.scheduler.stop()
|
|
if self.tts:
|
|
await self.tts.stop()
|
|
|
|
async def serve(self, host: str, port: int):
|
|
"""Start listening; returns the websockets server. Loopback addresses only."""
|
|
if not is_loopback(host):
|
|
raise ValueError("the voice worker only listens on a loopback address")
|
|
from websockets.asyncio.server import serve
|
|
await self.start()
|
|
return await serve(self.handle, host, port, max_size=2 ** 20)
|
|
|
|
async def handle(self, ws) -> None:
|
|
try:
|
|
first = await asyncio.wait_for(ws.recv(), self.hello_timeout_s)
|
|
hello = json.loads(first) if isinstance(first, str) else None
|
|
except (asyncio.TimeoutError, ValueError):
|
|
hello = None
|
|
except Exception:
|
|
return
|
|
if not isinstance(hello, dict) or hello.get("type") != "hello":
|
|
debug_log("connection refused: no hello", "voice_worker")
|
|
await ws.close(CLOSE_BAD_HELLO, "hello expected")
|
|
return
|
|
token = hello.get("token")
|
|
if not isinstance(token, str) or not hmac.compare_digest(token.encode(), self.token.encode()):
|
|
debug_log("connection refused: bad token", "voice_worker")
|
|
await ws.close(CLOSE_BAD_TOKEN, "bad token")
|
|
return
|
|
try:
|
|
session = Session(self, ws, next(self._ids), hello)
|
|
except ValueError as e:
|
|
await ws.close(CLOSE_BAD_HELLO, str(e)[:100])
|
|
return
|
|
await session.run()
|
|
|
|
|
|
@dataclass
|
|
class _Speaker:
|
|
converter: InputConverter
|
|
segmenter: Segmenter
|
|
|
|
|
|
def _format(value, default: dict, rates: tuple[int, ...] | None) -> dict:
|
|
if value is None:
|
|
return dict(default)
|
|
if not isinstance(value, dict):
|
|
raise ValueError("format must be an object")
|
|
rate, channels = value.get("rate", default["rate"]), value.get("channels", default["channels"])
|
|
ok_rate = isinstance(rate, int) and (rate in rates if rates else 8000 <= rate <= 96000)
|
|
if not ok_rate or channels not in (1, 2) or isinstance(channels, bool):
|
|
raise ValueError("unsupported audio format")
|
|
return {"rate": rate, "channels": channels}
|
|
|
|
|
|
class Session:
|
|
def __init__(self, worker: VoiceWorker, ws, sid: int, hello: dict):
|
|
self.worker = worker
|
|
self.ws = ws
|
|
self.sid = sid
|
|
self.audio = _format(hello.get("audio"), {"rate": 48000, "channels": 2}, (16000, 48000))
|
|
self.tts_format = _format(hello.get("tts"), {"rate": 48000, "channels": 2}, None)
|
|
self.settings = SttSettings()
|
|
stt = hello.get("stt")
|
|
if stt is not None:
|
|
if not isinstance(stt, dict):
|
|
raise ValueError("stt must be an object")
|
|
self.settings.update(stt)
|
|
self.memory = MemorySettings()
|
|
memory = hello.get("memory")
|
|
if memory is not None:
|
|
if not isinstance(memory, dict):
|
|
raise ValueError("memory must be an object")
|
|
self.memory.update(memory)
|
|
self.voice_channel: str | None = None
|
|
self.chat_channels: list[dict] = []
|
|
self.chat_waiting: dict[str, asyncio.Future] = {}
|
|
self._chat_ids = itertools.count(1)
|
|
self.names: dict[str, str] = {}
|
|
self.speakers: dict[str, _Speaker] = {}
|
|
self.tts_requests: dict[str, TtsRequest] = {}
|
|
self.out: asyncio.Queue = asyncio.Queue()
|
|
self.closed = False
|
|
|
|
# --- outbound -------------------------------------------------------------------------------
|
|
def send(self, msg) -> None:
|
|
if not self.closed:
|
|
self.out.put_nowait(msg if isinstance(msg, bytes) else json.dumps(msg, ensure_ascii=False))
|
|
|
|
def error(self, code: str, message: str, **extra) -> None:
|
|
self.send({"type": "error", "code": code, "message": message, **extra})
|
|
|
|
async def _writer(self) -> None:
|
|
while True:
|
|
await self.ws.send(await self.out.get())
|
|
|
|
# --- lifecycle ------------------------------------------------------------------------------
|
|
async def run(self) -> None:
|
|
w = self.worker
|
|
debug_log(f"session {self.sid} connected audio={self.audio} tts={self.tts_format}", "voice_worker")
|
|
self.send({"type": "hello_ok", "stt_ready": w.scheduler is not None, "tts_ready": w.tts is not None,
|
|
"audio": self.audio, "tts": self.tts_format, "stt": self.settings.as_dict()})
|
|
writer = asyncio.create_task(self._writer())
|
|
ticker = asyncio.create_task(self._ticker())
|
|
try:
|
|
async for frame in self.ws:
|
|
if isinstance(frame, bytes):
|
|
self._on_binary(frame)
|
|
else:
|
|
self._on_text(frame)
|
|
except Exception as e: # connection dropped
|
|
debug_log(f"session {self.sid} receive ended: {type(e).__name__}", "voice_worker")
|
|
finally:
|
|
self.closed = True
|
|
self._leave_voice()
|
|
for fut in self.chat_waiting.values():
|
|
if not fut.done():
|
|
fut.set_result({"ok": False, "error": "the app disconnected"})
|
|
ticker.cancel()
|
|
writer.cancel()
|
|
if w.scheduler:
|
|
w.scheduler.drop_partials(lambda key: key[0] == self.sid)
|
|
if w.tts:
|
|
for req in list(self.tts_requests.values()):
|
|
w.tts.cancel(req)
|
|
debug_log(f"session {self.sid} closed", "voice_worker")
|
|
|
|
async def _ticker(self) -> None:
|
|
while True:
|
|
await asyncio.sleep(self.worker.tick_s)
|
|
now = time.monotonic()
|
|
for speaker, st in list(self.speakers.items()):
|
|
self._handle_events(speaker, st, st.segmenter.tick(now))
|
|
|
|
# --- STT ------------------------------------------------------------------------------------
|
|
def _on_binary(self, frame: bytes) -> None:
|
|
if not frame or frame[0] != KIND_AUDIO:
|
|
self.error("bad_frame", "unknown binary frame kind")
|
|
return
|
|
try:
|
|
speaker, pcm = decode_audio_frame(frame)
|
|
except ProtocolError as e:
|
|
self.error("bad_frame", str(e))
|
|
return
|
|
st = self.speakers.get(speaker)
|
|
if st is None:
|
|
if len(self.speakers) >= MAX_SPEAKERS:
|
|
self.error("bad_frame", f"more than {MAX_SPEAKERS} speakers")
|
|
return
|
|
st = _Speaker(InputConverter(self.audio["rate"], self.audio["channels"]),
|
|
Segmenter(self.worker.vad_factory(), self.settings))
|
|
self.speakers[speaker] = st
|
|
samples = st.converter.push(pcm)
|
|
self._handle_events(speaker, st, st.segmenter.push(samples, time.monotonic()))
|
|
|
|
def _opts(self, beam: int) -> DecodeOptions:
|
|
return DecodeOptions(beam=beam, language=self.settings.language, hotwords=self.settings.hotwords)
|
|
|
|
def _handle_events(self, speaker: str, st: _Speaker, events: list) -> None:
|
|
sched = self.worker.scheduler
|
|
for ev in events:
|
|
if isinstance(ev, SpeechStart):
|
|
self.send({"type": "speech_start", "speaker": speaker, "utt": ev.utt})
|
|
elif isinstance(ev, Partial):
|
|
if sched and self.settings.partials:
|
|
sched.submit_partial((self.sid, speaker), loudness_equalise(ev.audio, ev.voiced), self._opts(1),
|
|
self._partial_cb(speaker, st, ev.utt))
|
|
elif isinstance(ev, Final):
|
|
debug_log(f"utterance end speaker={speaker} utt={ev.utt} reason={ev.reason} "
|
|
f"ms={len(ev.audio) * 1000 // SR}", "voice_worker")
|
|
# Where, when and by whom it was said, fixed now: decoding can take a while, and a
|
|
# move or a late joiner meanwhile must not change who heard it.
|
|
said = (self.voice_channel, self.names.get(speaker, speaker),
|
|
time.time() - (time.monotonic() - ev.decided_at))
|
|
if sched is None:
|
|
self._send_final(speaker, ev, SttResult("", 0, "stt unavailable"), said)
|
|
else:
|
|
sched.submit_final((self.sid, speaker), loudness_equalise(ev.audio, ev.voiced),
|
|
self._opts(self.settings.final_beam),
|
|
lambda res, ev=ev, said=said: self._send_final(speaker, ev, res, said))
|
|
|
|
def _partial_cb(self, speaker: str, st: _Speaker, utt: int):
|
|
def cb(res: SttResult) -> None:
|
|
# Drop it when the utterance already ended or captions were turned off meanwhile.
|
|
if st.segmenter.open_utt == utt and self.settings.partials and not res.error:
|
|
self.send({"type": "partial", "speaker": speaker, "utt": utt, "text": res.text})
|
|
return cb
|
|
|
|
def _send_final(self, speaker: str, ev: Final, res: SttResult, said: tuple) -> None:
|
|
channel, name, ts = said
|
|
store = self.worker.history
|
|
if store is not None and channel and res.text.strip():
|
|
store.add_voice(channel, speaker, name, res.text, ts)
|
|
self.send({"type": "final", "speaker": speaker, "utt": ev.utt, "text": res.text,
|
|
"end_reason": ev.reason, "audio_ms": len(ev.audio) * 1000 // SR,
|
|
"wait_ms": int((time.monotonic() - ev.decided_at) * 1000), "stt_ms": res.stt_ms})
|
|
|
|
# --- text messages --------------------------------------------------------------------------
|
|
def _on_text(self, raw: str) -> None:
|
|
try:
|
|
msg = json.loads(raw)
|
|
except ValueError:
|
|
self.error("bad_message", "not JSON")
|
|
return
|
|
if not isinstance(msg, dict):
|
|
self.error("bad_message", "not an object")
|
|
return
|
|
kind = msg.get("type")
|
|
handler = {"speaking_end": self._on_speaking_end, "speaker_leave": self._on_speaker_leave,
|
|
"config": self._on_config, "tts": self._on_tts, "tts_cancel": self._on_tts_cancel,
|
|
"voice_join": self._on_voice_join, "voice_leave": lambda m: self._leave_voice(),
|
|
"member_join": self._on_member_join, "member_leave": self._on_member_leave,
|
|
"chat_message": self._on_chat_message, "chat_edit": self._on_chat_edit,
|
|
"chat_delete": self._on_chat_delete, "reply": self._on_reply,
|
|
"chat_channels": self._on_chat_channels, "chat_action_result": self._on_chat_action_result,
|
|
"memory_config": self._on_memory_config}.get(kind)
|
|
if handler is None:
|
|
self.error("bad_message", f"unknown type: {kind!r}")
|
|
return
|
|
handler(msg)
|
|
|
|
def _on_speaking_end(self, msg: dict) -> None:
|
|
st = self.speakers.get(msg.get("speaker"))
|
|
if st is not None:
|
|
self._handle_events(msg["speaker"], st, st.segmenter.speaking_end(time.monotonic()))
|
|
|
|
def _on_speaker_leave(self, msg: dict) -> None:
|
|
# The speaker left the voice channel: finish any open utterance, then free their place.
|
|
speaker = msg.get("speaker")
|
|
st = self.speakers.pop(speaker, None) if isinstance(speaker, str) else None
|
|
if st is not None:
|
|
self._handle_events(speaker, st, st.segmenter.speaking_end(time.monotonic()))
|
|
if self.worker.scheduler:
|
|
self.worker.scheduler.drop_partials(lambda key: key == (self.sid, speaker))
|
|
debug_log(f"speaker {speaker} left session {self.sid}", "voice_worker")
|
|
|
|
def _on_config(self, msg: dict) -> None:
|
|
changes = {k: v for k, v in msg.items() if k != "type"}
|
|
try:
|
|
self.settings.update(changes)
|
|
except ValueError as e:
|
|
self.error("bad_message", str(e))
|
|
return
|
|
if not self.settings.partials and self.worker.scheduler:
|
|
self.worker.scheduler.drop_partials(lambda key: key[0] == self.sid)
|
|
debug_log(f"session {self.sid} settings changed: {sorted(changes)}", "voice_worker")
|
|
|
|
def _on_tts(self, msg: dict) -> None:
|
|
rid, text, more = msg.get("id"), msg.get("text", ""), msg.get("more", False)
|
|
if not isinstance(rid, str) or not 0 < len(rid.encode()) < 256 or not isinstance(text, str) \
|
|
or not isinstance(more, bool):
|
|
self.error("bad_message", "tts needs a string id and text, and a boolean more")
|
|
return
|
|
tts = self.worker.tts
|
|
if tts is None:
|
|
self.error("tts_unavailable", "no TTS engine is loaded", id=rid)
|
|
return
|
|
req = self.tts_requests.get(rid)
|
|
if req is None:
|
|
req = TtsRequest(rid, self.tts_format["rate"], self._tts_chunk_cb(rid), self._tts_done,
|
|
self._tts_error)
|
|
self.tts_requests[rid] = req
|
|
elif req.closed:
|
|
self.error("bad_message", "this tts id is already closed", id=rid)
|
|
return
|
|
tts.feed(req, text, more)
|
|
|
|
def _on_tts_cancel(self, msg: dict) -> None:
|
|
req = self.tts_requests.get(msg.get("id"))
|
|
if req is not None and self.worker.tts:
|
|
self.worker.tts.cancel(req)
|
|
|
|
# --- history and brain ----------------------------------------------------------------------
|
|
def _strings(self, msg: dict, *keys: str) -> tuple | None:
|
|
values = tuple(msg.get(k) for k in keys)
|
|
if not all(isinstance(v, str) and v for v in values):
|
|
self.error("bad_message", f"{msg.get('type')} needs non-empty strings: {', '.join(keys)}",
|
|
**({"id": msg["id"]} if isinstance(msg.get("id"), str) else {}))
|
|
return None
|
|
return values
|
|
|
|
def _leave_voice(self) -> None:
|
|
if self.voice_channel and self.worker.history is not None:
|
|
self.worker.history.presence_close_channel(self.voice_channel, time.time())
|
|
if self.voice_channel:
|
|
debug_log(f"session {self.sid} left voice channel {self.voice_channel}", "voice_worker")
|
|
self.voice_channel = None
|
|
|
|
def _on_voice_join(self, msg: dict) -> None:
|
|
got = self._strings(msg, "channel")
|
|
if got is None:
|
|
return
|
|
self._leave_voice()
|
|
self.voice_channel = got[0]
|
|
debug_log(f"session {self.sid} in voice channel {self.voice_channel}", "voice_worker")
|
|
|
|
def _on_member_join(self, msg: dict) -> None:
|
|
got = self._strings(msg, "user", "name")
|
|
if got is None:
|
|
return
|
|
user, name = got
|
|
self.names[user] = name
|
|
if self.voice_channel and self.worker.history is not None:
|
|
self.worker.history.presence_join(user, self.voice_channel, time.time())
|
|
|
|
def _on_member_leave(self, msg: dict) -> None:
|
|
got = self._strings(msg, "user")
|
|
if got is not None and self.voice_channel and self.worker.history is not None:
|
|
self.worker.history.presence_leave(got[0], self.voice_channel, time.time())
|
|
|
|
def _on_chat_message(self, msg: dict) -> None:
|
|
got = self._strings(msg, "channel", "message_id", "author", "name")
|
|
ts, to, text = msg.get("ts"), msg.get("to"), msg.get("text")
|
|
if got is None:
|
|
return
|
|
if not isinstance(text, str) or isinstance(ts, bool) or not isinstance(ts, (int, float)) \
|
|
or (to is not None and not isinstance(to, str)):
|
|
self.error("bad_message", "chat_message needs text, a numeric ts and an optional string to")
|
|
return
|
|
if self.worker.history is not None:
|
|
channel, message_id, author, name = got
|
|
self.worker.history.add_chat(channel, message_id, author, name, text, float(ts), to_id=to)
|
|
|
|
def _on_chat_edit(self, msg: dict) -> None:
|
|
got = self._strings(msg, "channel", "message_id")
|
|
if got is None:
|
|
return
|
|
if not isinstance(msg.get("text"), str):
|
|
self.error("bad_message", "chat_edit needs text")
|
|
return
|
|
if self.worker.history is not None:
|
|
self.worker.history.edit_chat(got[0], got[1], msg["text"])
|
|
|
|
def _on_chat_delete(self, msg: dict) -> None:
|
|
got = self._strings(msg, "channel", "message_id")
|
|
if got is not None and self.worker.history is not None:
|
|
self.worker.history.delete_chat(*got)
|
|
|
|
_CHAT_FLAGS = ("read", "send", "edit", "react", "delete", "pin")
|
|
|
|
def _on_chat_channels(self, msg: dict) -> None:
|
|
channels = msg.get("channels")
|
|
ok = isinstance(channels, list) and all(
|
|
isinstance(c, dict) and all(isinstance(c.get(k), str) for k in ("id", "name", "description"))
|
|
and c["id"] and all(isinstance(c.get(f), bool) for f in self._CHAT_FLAGS) for c in channels)
|
|
if not ok:
|
|
self.error("bad_message", "chat_channels needs channels with id, name, description and "
|
|
"boolean read, send, edit, react, delete, pin")
|
|
return
|
|
self.chat_channels = [{k: c[k] for k in ("id", "name", "description", *self._CHAT_FLAGS)} for c in channels]
|
|
debug_log(f"session {self.sid} chat channels: {len(self.chat_channels)}", "voice_worker")
|
|
|
|
async def chat_action(self, action: str, channel: str, args: dict) -> dict:
|
|
"""Ask the app to do a chat action and wait for its answer (or a timeout)."""
|
|
aid = f"a{next(self._chat_ids)}"
|
|
fut = asyncio.get_running_loop().create_future()
|
|
self.chat_waiting[aid] = fut
|
|
self.send({"type": "chat_action", "id": aid, "action": action, "channel": channel,
|
|
**{k: v for k, v in args.items() if v is not None}})
|
|
debug_log(f"chat action {aid} {action} channel={channel}", "voice_worker")
|
|
try:
|
|
return await asyncio.wait_for(fut, self.worker.chat_timeout_s)
|
|
except asyncio.TimeoutError:
|
|
return {"ok": False, "error": "the app did not answer in time"}
|
|
finally:
|
|
self.chat_waiting.pop(aid, None)
|
|
|
|
def _on_chat_action_result(self, msg: dict) -> None:
|
|
fut = self.chat_waiting.get(msg.get("id"))
|
|
if fut is None or fut.done():
|
|
return
|
|
ids = msg.get("message_ids") or []
|
|
result = {"ok": msg.get("ok") is True}
|
|
if result["ok"]:
|
|
result["message_ids"] = [str(i) for i in ids if isinstance(i, (str, int))]
|
|
else:
|
|
result["error"] = str(msg.get("error") or "the app could not do it")
|
|
fut.set_result(result)
|
|
|
|
def _on_memory_config(self, msg: dict) -> None:
|
|
changes = {k: v for k, v in msg.items() if k != "type"}
|
|
try:
|
|
self.memory.update(changes)
|
|
except ValueError as e:
|
|
self.error("bad_message", str(e))
|
|
return
|
|
debug_log(f"session {self.sid} memory settings changed: {sorted(changes)}", "voice_worker")
|
|
|
|
def _on_reply(self, msg: dict) -> None:
|
|
got = self._strings(msg, "id", "user", "text")
|
|
if got is None:
|
|
return
|
|
rid, user, text = got
|
|
via, speak = msg.get("via"), msg.get("speak", False)
|
|
if via not in ("voice", "chat") or not isinstance(speak, bool) or len(rid.encode()) > 255:
|
|
self.error("bad_message", "reply needs via voice|chat, a boolean speak and an id under 256 bytes",
|
|
id=rid)
|
|
return
|
|
if self.worker.brain is None:
|
|
self.error("brain_unavailable", "the worker was started without --brain", id=rid)
|
|
return
|
|
asyncio.get_running_loop().create_task(self._reply(rid, user, text, via, speak))
|
|
|
|
async def _reply(self, rid: str, user: str, text: str, via: str, speak: bool) -> None:
|
|
w = self.worker
|
|
channel = self.voice_channel
|
|
async with w.brain_lock:
|
|
if self.closed:
|
|
return
|
|
now = time.time()
|
|
if w.history is not None:
|
|
m = self.memory
|
|
history = await asyncio.to_thread(
|
|
w.history.context, user, text, now, m.scope(now), minutes=m.recent_minutes,
|
|
max_records=m.recent_records, limit=m.matches, budget_tokens=m.budget_tokens)
|
|
else:
|
|
from jarvis.memory.history import HistoryContext
|
|
history = HistoryContext()
|
|
turn = w.reply_turns[user] = w.reply_turns.get(user, 0) + 1
|
|
from .chat_bridge import ChatBridge
|
|
chat = ChatBridge(self, asyncio.get_running_loop(), turn, w.pending_deletes.setdefault(user, {})) \
|
|
if self.chat_channels else None
|
|
try:
|
|
answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat)
|
|
except Exception as e: # the brain must never take the session down
|
|
debug_log(f"brain reply {rid} failed: {type(e).__name__}", "voice_worker")
|
|
answer = None
|
|
answer = (answer or "").strip()
|
|
self.send({"type": "reply_text", "id": rid, "text": answer, "failed": not answer})
|
|
if not answer:
|
|
return
|
|
if via == "voice" and channel and w.history is not None:
|
|
w.history.add_reply(channel, user, answer, time.time())
|
|
if speak:
|
|
self._on_tts({"type": "tts", "id": rid, "text": answer, "more": False})
|
|
|
|
def _tts_chunk_cb(self, rid: str):
|
|
def cb(seq: int, pcm: np.ndarray) -> None:
|
|
pcm = np.asarray(pcm, dtype="<i2")
|
|
if self.tts_format["channels"] == 2:
|
|
pcm = np.repeat(pcm, 2)
|
|
self.send(encode_tts_frame(rid, seq, pcm.tobytes()))
|
|
return cb
|
|
|
|
def _tts_done(self, req: TtsRequest) -> None:
|
|
self.tts_requests.pop(req.id, None)
|
|
self.send({"type": "tts_done", "id": req.id, "chunks": req.chunks, "cancelled": req.cancelled,
|
|
"failed": req.failed})
|
|
|
|
def _tts_error(self, req: TtsRequest, message: str) -> None:
|
|
self.error("tts_failed", message, id=req.id)
|