- Playback tells the worker how much of each request was played (tts_played); the worker keeps a spoken answer's recording only that far (an answer stopped mid-way no longer keeps its unheard rest, and is marked stopped); nothing played or never reported: no file. - Old recording day folders are also removed with the first recording of each new day. - response.interrupt off, latest: a request whose final comes after a newer one's is replaced instead of replacing it. - botBrowser presses play on a page's paused main player also after open, not only after a click. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
997 lines
49 KiB
Python
997 lines
49 KiB
Python
"""Local WebSocket server: one session per connection (protocol in voice_worker.spec.md)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import contextlib
|
|
import hmac
|
|
import ipaddress
|
|
import itertools
|
|
import json
|
|
import secrets
|
|
import time
|
|
from dataclasses import dataclass
|
|
from typing import Callable
|
|
|
|
import numpy as np
|
|
|
|
from .log import debug_log
|
|
|
|
from .audio import InputConverter, loudness_equalise
|
|
from .brain import Brain, MemorySettings
|
|
from .protocol import KIND_AUDIO, ProtocolError, decode_audio_frame, encode_tts_frame
|
|
from .scheduler import DecodeOptions, SttResult, SttScheduler, Transcriber
|
|
from .segmenter import SR, Final, Partial, Segmenter, SpeechStart, VadStream
|
|
from .settings import SttSettings
|
|
from .tts import TtsEngine, TtsRequest, TtsService
|
|
|
|
CLOSE_BAD_HELLO = 4400
|
|
CLOSE_BAD_TOKEN = 4401
|
|
MAX_SPEAKERS = 32
|
|
|
|
|
|
def is_loopback(host: str) -> bool:
|
|
if host == "localhost":
|
|
return True
|
|
try:
|
|
return ipaddress.ip_address(host).is_loopback
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
# How long a request waits for the LLM to come back from a switch (a restart takes 30-50 s; the
|
|
# first game switch also measures once).
|
|
LLM_WAIT_S = 180.0
|
|
|
|
|
|
class _LlmPaused(Exception):
|
|
"""The app paused the LLM (a game runs): nothing is answered until it is ready again."""
|
|
|
|
|
|
# The asker's standing instructions the app sends with a reply (personal_notes.spec.md).
|
|
MAX_NOTES = 20
|
|
MAX_NOTE_CHARS = 300
|
|
# How long answer traces stay (history.spec.md, "Traces"); removed when the worker starts.
|
|
TRACE_KEEP_DAYS = 30
|
|
|
|
|
|
class _LiveTrace(list):
|
|
"""A reply's trace (history.spec.md, "Traces"): stored as soon as the request arrives and again
|
|
as each step is added, so the 대화 screen fills it in while the answer is made. The engine adds
|
|
steps from the brain's thread. Storing never stops the answer."""
|
|
|
|
def __init__(self, store, via: str, channel: str | None, user: str, question: str):
|
|
super().__init__()
|
|
self.store, self.began, self.id = store, time.time(), None
|
|
if store is not None:
|
|
self.id = self._keep(store.start_trace, via, channel, user, question, ts=self.began)
|
|
|
|
@staticmethod
|
|
def _keep(write, *args, **kw):
|
|
try:
|
|
return write(*args, **kw)
|
|
except Exception as e: # noqa: BLE001 - a trace is a view, not part of the answer
|
|
debug_log(f"trace not kept: {type(e).__name__}", "llm")
|
|
return None
|
|
|
|
def append(self, step) -> None:
|
|
super().append(step)
|
|
if self.id is not None:
|
|
self._keep(self.store.trace_steps, self.id, list(self))
|
|
|
|
def finish(self, answer: str) -> None:
|
|
if self.id is not None:
|
|
self._keep(self.store.finish_trace, self.id, answer, list(self),
|
|
ms=round((time.time() - self.began) * 1000))
|
|
self.id = None
|
|
|
|
|
|
class VoiceWorker:
|
|
def _partials_held(self) -> bool:
|
|
return self.llm_game and self.answering > 0
|
|
|
|
def __init__(self, token: str, transcriber: Transcriber | None, vad_factory: Callable[[], VadStream],
|
|
tts_engine: TtsEngine | None = None, hello_timeout_s: float = 5.0, tick_s: float = 0.05,
|
|
history=None, brain: Brain | None = None, chat_timeout_s: float = 15.0,
|
|
delivery_timeout_s: float = 30.0, recorder=None):
|
|
if not token:
|
|
raise ValueError("a connection token is required")
|
|
self.token = token
|
|
self.transcriber = transcriber
|
|
self.vad_factory = vad_factory
|
|
self.tts_engine = tts_engine
|
|
self.hello_timeout_s = hello_timeout_s
|
|
self.tick_s = tick_s
|
|
self.history = history # jarvis.memory.history.HistoryStore or None
|
|
self.recorder = recorder # recordings.Recorder (voice.record on) or None
|
|
self.brain = brain
|
|
# One reply at a time per lane: voice answers never wait behind another person's chat answer
|
|
# (conversation.md: with an "answer everything" chat channel the LLM takes two or more
|
|
# requests at once). One person's replies stay one at a time across lanes, so their dialogue
|
|
# memory and delete confirmations ("the very next message") follow the order they asked.
|
|
self.brain_lanes = {"voice": asyncio.Lock(), "chat": asyncio.Lock()}
|
|
self.brain_people: dict[str, asyncio.Lock] = {}
|
|
# The LLM's state from the app (voice_worker.spec.md, "LLM switches"): while it switches or
|
|
# is paused no new brain work starts; the brain work running when a switch is announced
|
|
# finishes first, then the app is told the brain is idle.
|
|
self.llm_state = "ready"
|
|
self.llm_ready = asyncio.Event()
|
|
self.llm_ready.set()
|
|
self.llm_notice: str | None = None
|
|
self.llm_noticed = False
|
|
self.llm_active = 0
|
|
self.llm_idle_to = None # the session to tell once nothing runs
|
|
self.llm_switch_id = None # the switch to name in llm_idle
|
|
# In a game (the app says so with `ready`), partial passes wait while an answer is written
|
|
# (M0 gamemode_tune: the LLM answers 1.7 times faster); finals never wait.
|
|
self.llm_game = False
|
|
self.answering = 0
|
|
self.chat_timeout_s = chat_timeout_s
|
|
self.delivery_timeout_s = delivery_timeout_s # a chat answer the app never reports: not posted
|
|
# chat_action ids are unique across connections and worker runs, so a late answer from an
|
|
# earlier connection (or an earlier worker) never completes a newer action.
|
|
self.chat_action_prefix = secrets.token_hex(4)
|
|
# Per person: how many replies they had (a delete is confirmed only in the very next one),
|
|
# the deletes they were asked to confirm (chat_channels.spec.md) and how many of their
|
|
# replies have not reached them yet (spoken, or posted by the app). A turn number is skipped
|
|
# for a request that arrived before the person's previous answer reached them, and after an
|
|
# answer that never did, so a confirmation asked in that answer lapses.
|
|
self.reply_turns: dict[str, int] = {}
|
|
self.pending_deletes: dict[str, dict] = {}
|
|
self.replies_unanswered: dict[str, int] = {}
|
|
self.scheduler: SttScheduler | None = None
|
|
self.tts: TtsService | None = None
|
|
self._ids = itertools.count(1)
|
|
|
|
async def start(self) -> None:
|
|
if self.history is not None:
|
|
# A killed worker never closed its sessions' presence; nobody is heard until the app
|
|
# reports who is there again.
|
|
n = self.history.close_open_presence(time.time())
|
|
if n:
|
|
debug_log(f"closed {n} presence intervals left open by a previous run", "voice_worker")
|
|
n = self.history.close_unfinished_traces()
|
|
if n:
|
|
debug_log(f"closed {n} answer traces left unfinished by a previous run", "voice_worker")
|
|
n = self.history.purge_traces_older_than(time.time() - TRACE_KEEP_DAYS * 86400)
|
|
if n:
|
|
debug_log(f"removed {n} answer traces older than {TRACE_KEEP_DAYS} days", "voice_worker")
|
|
if self.transcriber is not None:
|
|
self.scheduler = SttScheduler(self.transcriber, partials_held=self._partials_held)
|
|
self.scheduler.start()
|
|
if self.tts_engine is not None:
|
|
self.tts = TtsService(self.tts_engine)
|
|
self.tts.start()
|
|
|
|
async def stop(self) -> None:
|
|
if self.scheduler:
|
|
await self.scheduler.stop()
|
|
if self.tts:
|
|
await self.tts.stop()
|
|
|
|
async def serve(self, host: str, port: int):
|
|
"""Start listening; returns the websockets server. Loopback addresses only."""
|
|
if not is_loopback(host):
|
|
raise ValueError("the voice worker only listens on a loopback address")
|
|
from websockets.asyncio.server import serve
|
|
await self.start()
|
|
return await serve(self.handle, host, port, max_size=4 * 2 ** 20) # pictures (screen_action_result)
|
|
|
|
async def handle(self, ws) -> None:
|
|
try:
|
|
first = await asyncio.wait_for(ws.recv(), self.hello_timeout_s)
|
|
hello = json.loads(first) if isinstance(first, str) else None
|
|
except (asyncio.TimeoutError, ValueError):
|
|
hello = None
|
|
except Exception:
|
|
return
|
|
if not isinstance(hello, dict) or hello.get("type") != "hello":
|
|
debug_log("connection refused: no hello", "voice_worker")
|
|
await ws.close(CLOSE_BAD_HELLO, "hello expected")
|
|
return
|
|
token = hello.get("token")
|
|
if not isinstance(token, str) or not hmac.compare_digest(token.encode(), self.token.encode()):
|
|
debug_log("connection refused: bad token", "voice_worker")
|
|
await ws.close(CLOSE_BAD_TOKEN, "bad token")
|
|
return
|
|
try:
|
|
session = Session(self, ws, next(self._ids), hello)
|
|
except ValueError as e:
|
|
await ws.close(CLOSE_BAD_HELLO, str(e)[:100])
|
|
return
|
|
await session.run()
|
|
|
|
|
|
@dataclass
|
|
class _Speaker:
|
|
converter: InputConverter
|
|
segmenter: Segmenter
|
|
|
|
|
|
def _format(value, default: dict, rates: tuple[int, ...] | None) -> dict:
|
|
if value is None:
|
|
return dict(default)
|
|
if not isinstance(value, dict):
|
|
raise ValueError("format must be an object")
|
|
rate, channels = value.get("rate", default["rate"]), value.get("channels", default["channels"])
|
|
ok_rate = isinstance(rate, int) and (rate in rates if rates else 8000 <= rate <= 96000)
|
|
if not ok_rate or channels not in (1, 2) or isinstance(channels, bool):
|
|
raise ValueError("unsupported audio format")
|
|
return {"rate": rate, "channels": channels}
|
|
|
|
|
|
class Session:
|
|
def __init__(self, worker: VoiceWorker, ws, sid: int, hello: dict):
|
|
self.worker = worker
|
|
self.ws = ws
|
|
self.sid = sid
|
|
self.audio = _format(hello.get("audio"), {"rate": 48000, "channels": 2}, (16000, 48000))
|
|
self.tts_format = _format(hello.get("tts"), {"rate": 48000, "channels": 2}, None)
|
|
self.settings = SttSettings()
|
|
stt = hello.get("stt")
|
|
if stt is not None:
|
|
if not isinstance(stt, dict):
|
|
raise ValueError("stt must be an object")
|
|
self.settings.update(stt)
|
|
self.memory = MemorySettings()
|
|
memory = hello.get("memory")
|
|
if memory is not None:
|
|
if not isinstance(memory, dict):
|
|
raise ValueError("memory must be an object")
|
|
self.memory.update(memory)
|
|
self.voice_channel: str | None = None
|
|
self.chat_channels: list[dict] = []
|
|
self.chat_waiting: dict[str, asyncio.Future] = {}
|
|
self._chat_ids = itertools.count(1)
|
|
self.names: dict[str, str] = {}
|
|
self.speakers: dict[str, _Speaker] = {}
|
|
self.tts_requests: dict[str, TtsRequest] = {}
|
|
# With recordings on: each spoken answer's text and mono samples, kept until it is made and the
|
|
# app said how much of it was played (`tts_played`).
|
|
self.tts_recorded: dict[str, dict] = {}
|
|
# Spoken replies whose speech was cancelled while the brain was still answering.
|
|
self.cancelled_replies: set[str] = set()
|
|
self.thinking_replies: set[str] = set()
|
|
# user -> the newest call text waiting for that user's running prepare to finish (or None).
|
|
self.preparing: dict[str, str | None] = {}
|
|
self.last_prepared: dict[str, str] = {} # user -> the text of their last prepare
|
|
# Answers sent but not yet known to have reached the person: id -> (user, turn, timer).
|
|
self.deliveries: dict[str, tuple[str, int, asyncio.TimerHandle | None]] = {}
|
|
self._handed_over: dict[str, bool] = {} # replies whose end is counted by _delivered
|
|
self.out: asyncio.Queue = asyncio.Queue()
|
|
self.closed = False
|
|
|
|
# --- outbound -------------------------------------------------------------------------------
|
|
def send(self, msg) -> None:
|
|
if not self.closed:
|
|
self.out.put_nowait(msg if isinstance(msg, bytes) else json.dumps(msg, ensure_ascii=False))
|
|
|
|
def error(self, code: str, message: str, **extra) -> None:
|
|
self.send({"type": "error", "code": code, "message": message, **extra})
|
|
|
|
async def _writer(self) -> None:
|
|
while True:
|
|
await self.ws.send(await self.out.get())
|
|
|
|
# --- lifecycle ------------------------------------------------------------------------------
|
|
async def run(self) -> None:
|
|
w = self.worker
|
|
debug_log(f"session {self.sid} connected audio={self.audio} tts={self.tts_format}", "voice_worker")
|
|
self.send({"type": "hello_ok", "stt_ready": w.scheduler is not None, "tts_ready": w.tts is not None,
|
|
"audio": self.audio, "tts": self.tts_format, "stt": self.settings.as_dict()})
|
|
writer = asyncio.create_task(self._writer())
|
|
ticker = asyncio.create_task(self._ticker())
|
|
try:
|
|
async for frame in self.ws:
|
|
if isinstance(frame, bytes):
|
|
self._on_binary(frame)
|
|
else:
|
|
self._on_text(frame)
|
|
except Exception as e: # connection dropped
|
|
debug_log(f"session {self.sid} receive ended: {type(e).__name__}", "voice_worker")
|
|
finally:
|
|
self.closed = True
|
|
self._leave_voice()
|
|
for fut in self.chat_waiting.values():
|
|
if not fut.done():
|
|
fut.set_result({"ok": False, "error": "the app disconnected"})
|
|
for rid in list(self.deliveries):
|
|
self._delivered(rid, False)
|
|
ticker.cancel()
|
|
writer.cancel()
|
|
if w.scheduler:
|
|
w.scheduler.drop_partials(lambda key: key[0] == self.sid)
|
|
if w.tts:
|
|
for req in list(self.tts_requests.values()):
|
|
w.tts.cancel(req)
|
|
debug_log(f"session {self.sid} closed", "voice_worker")
|
|
|
|
async def _ticker(self) -> None:
|
|
while True:
|
|
await asyncio.sleep(self.worker.tick_s)
|
|
now = time.monotonic()
|
|
for speaker, st in list(self.speakers.items()):
|
|
self._handle_events(speaker, st, st.segmenter.tick(now))
|
|
|
|
# --- STT ------------------------------------------------------------------------------------
|
|
def _on_binary(self, frame: bytes) -> None:
|
|
if not frame or frame[0] != KIND_AUDIO:
|
|
self.error("bad_frame", "unknown binary frame kind")
|
|
return
|
|
try:
|
|
speaker, pcm = decode_audio_frame(frame)
|
|
except ProtocolError as e:
|
|
self.error("bad_frame", str(e))
|
|
return
|
|
st = self.speakers.get(speaker)
|
|
if st is None:
|
|
if len(self.speakers) >= MAX_SPEAKERS:
|
|
self.error("bad_frame", f"more than {MAX_SPEAKERS} speakers")
|
|
return
|
|
st = _Speaker(InputConverter(self.audio["rate"], self.audio["channels"]),
|
|
Segmenter(self.worker.vad_factory(), self.settings))
|
|
self.speakers[speaker] = st
|
|
samples = st.converter.push(pcm)
|
|
self._handle_events(speaker, st, st.segmenter.push(samples, time.monotonic()))
|
|
|
|
def _opts(self, beam: int) -> DecodeOptions:
|
|
return DecodeOptions(beam=beam, language=self.settings.language, hotwords=self.settings.hotwords)
|
|
|
|
def _handle_events(self, speaker: str, st: _Speaker, events: list) -> None:
|
|
sched = self.worker.scheduler
|
|
for ev in events:
|
|
if isinstance(ev, SpeechStart):
|
|
self.send({"type": "speech_start", "speaker": speaker, "utt": ev.utt})
|
|
elif isinstance(ev, Partial):
|
|
if sched and self.settings.partials:
|
|
sched.submit_partial((self.sid, speaker), loudness_equalise(ev.audio, ev.voiced), self._opts(1),
|
|
self._partial_cb(speaker, st, ev.utt))
|
|
elif isinstance(ev, Final):
|
|
debug_log(f"utterance end speaker={speaker} utt={ev.utt} reason={ev.reason} "
|
|
f"ms={len(ev.audio) * 1000 // SR}", "stt")
|
|
# Where, when and by whom it was said, fixed now: decoding can take a while, and a
|
|
# move or a late joiner meanwhile must not change who heard it.
|
|
said = (self.voice_channel, self.names.get(speaker, speaker),
|
|
time.time() - (time.monotonic() - ev.decided_at))
|
|
if sched is None:
|
|
self._send_final(speaker, ev, SttResult("", 0, "stt unavailable"), said)
|
|
else:
|
|
sched.submit_final((self.sid, speaker), loudness_equalise(ev.audio, ev.voiced),
|
|
self._opts(self.settings.final_beam),
|
|
lambda res, ev=ev, said=said: self._send_final(speaker, ev, res, said))
|
|
|
|
def _partial_cb(self, speaker: str, st: _Speaker, utt: int):
|
|
def cb(res: SttResult) -> None:
|
|
# Drop it when the utterance already ended or captions were turned off meanwhile.
|
|
if st.segmenter.open_utt == utt and self.settings.partials and not res.error:
|
|
self.send({"type": "partial", "speaker": speaker, "utt": utt, "text": res.text})
|
|
return cb
|
|
|
|
def _send_final(self, speaker: str, ev: Final, res: SttResult, said: tuple) -> None:
|
|
channel, name, ts = said
|
|
store = self.worker.history
|
|
if store is not None and channel and res.text.strip():
|
|
store.add_voice(channel, speaker, name, res.text, ts)
|
|
if self.worker.recorder is not None:
|
|
self.worker.recorder.said(ts, speaker, name, channel, res.text, ev.audio, SR)
|
|
self.send({"type": "final", "speaker": speaker, "utt": ev.utt, "text": res.text,
|
|
"end_reason": ev.reason, "audio_ms": len(ev.audio) * 1000 // SR,
|
|
"wait_ms": int((time.monotonic() - ev.decided_at) * 1000), "stt_ms": res.stt_ms})
|
|
|
|
# --- text messages --------------------------------------------------------------------------
|
|
def _on_text(self, raw: str) -> None:
|
|
try:
|
|
msg = json.loads(raw)
|
|
except ValueError:
|
|
self.error("bad_message", "not JSON")
|
|
return
|
|
if not isinstance(msg, dict):
|
|
self.error("bad_message", "not an object")
|
|
return
|
|
kind = msg.get("type")
|
|
handler = {"speaking_end": self._on_speaking_end, "speaker_leave": self._on_speaker_leave,
|
|
"config": self._on_config, "tts": self._on_tts, "tts_cancel": self._on_tts_cancel,
|
|
"tts_speed": self._on_tts_speed, "tts_played": self._on_tts_played,
|
|
"voice_join": self._on_voice_join, "voice_leave": lambda m: self._leave_voice(),
|
|
"member_join": self._on_member_join, "member_leave": self._on_member_leave,
|
|
"chat_message": self._on_chat_message, "chat_edit": self._on_chat_edit,
|
|
"chat_delete": self._on_chat_delete, "reply": self._on_reply,
|
|
"addressed": self._on_addressed, "rank": self._on_rank,
|
|
"chat_channels": self._on_chat_channels, "chat_action_result": self._on_chat_action_result,
|
|
"schedule_action_result": self._on_app_action_result,
|
|
"note_action_result": self._on_app_action_result,
|
|
"screen_action_result": self._on_app_action_result,
|
|
"memory_config": self._on_memory_config, "reply_delivered": self._on_reply_delivered,
|
|
"prepare": self._on_prepare, "llm": self._on_llm}.get(kind)
|
|
if handler is None:
|
|
self.error("bad_message", f"unknown type: {kind!r}")
|
|
return
|
|
handler(msg)
|
|
|
|
def _on_speaking_end(self, msg: dict) -> None:
|
|
st = self.speakers.get(msg.get("speaker"))
|
|
if st is not None:
|
|
self._handle_events(msg["speaker"], st, st.segmenter.speaking_end(time.monotonic()))
|
|
|
|
def _on_speaker_leave(self, msg: dict) -> None:
|
|
# The speaker left the voice channel: finish any open utterance, then free their place.
|
|
speaker = msg.get("speaker")
|
|
st = self.speakers.pop(speaker, None) if isinstance(speaker, str) else None
|
|
if st is not None:
|
|
self._handle_events(speaker, st, st.segmenter.speaking_end(time.monotonic()))
|
|
if self.worker.scheduler:
|
|
self.worker.scheduler.drop_partials(lambda key: key == (self.sid, speaker))
|
|
debug_log(f"speaker {speaker} left session {self.sid}", "voice_worker")
|
|
|
|
def _on_config(self, msg: dict) -> None:
|
|
changes = {k: v for k, v in msg.items() if k != "type"}
|
|
try:
|
|
self.settings.update(changes)
|
|
except ValueError as e:
|
|
self.error("bad_message", str(e))
|
|
return
|
|
if not self.settings.partials and self.worker.scheduler:
|
|
self.worker.scheduler.drop_partials(lambda key: key[0] == self.sid)
|
|
debug_log(f"session {self.sid} settings changed: {sorted(changes)}", "voice_worker")
|
|
|
|
def _on_tts_speed(self, msg: dict) -> None:
|
|
speed = msg.get("speed")
|
|
if isinstance(speed, bool) or not isinstance(speed, (int, float)) or not 0.5 <= speed <= 2.0:
|
|
self.error("bad_message", "tts_speed needs a speed from 0.5 to 2.0")
|
|
return
|
|
if self.worker.tts_engine is not None:
|
|
self.worker.tts_engine.speed = float(speed)
|
|
debug_log(f"tts speed {speed}", "tts")
|
|
|
|
def _on_tts(self, msg: dict) -> None:
|
|
rid, text, more = msg.get("id"), msg.get("text", ""), msg.get("more", False)
|
|
if not isinstance(rid, str) or not 0 < len(rid.encode()) < 256 or not isinstance(text, str) \
|
|
or not isinstance(more, bool):
|
|
self.error("bad_message", "tts needs a string id and text, and a boolean more")
|
|
return
|
|
tts = self.worker.tts
|
|
if tts is None:
|
|
self.error("tts_unavailable", "no TTS engine is loaded", id=rid)
|
|
self._delivered(rid, False)
|
|
return
|
|
req = self.tts_requests.get(rid)
|
|
if req is None:
|
|
req = TtsRequest(rid, self.tts_format["rate"], self._tts_chunk_cb(rid), self._tts_done,
|
|
self._tts_error)
|
|
self.tts_requests[rid] = req
|
|
elif req.closed:
|
|
self.error("bad_message", "this tts id is already closed", id=rid)
|
|
return
|
|
if self.worker.recorder is not None:
|
|
rec = self.tts_recorded.setdefault(rid, {"text": "", "pcm": [], "at": None, "rate": req.rate,
|
|
"made": False, "played": None})
|
|
rec["text"] += text
|
|
while len(self.tts_recorded) > 50: # never reported as played: not kept forever
|
|
self.tts_recorded.pop(next(iter(self.tts_recorded)))
|
|
tts.feed(req, text, more)
|
|
|
|
def _on_tts_cancel(self, msg: dict) -> None:
|
|
if msg.get("id") in self.thinking_replies:
|
|
self.cancelled_replies.add(msg["id"])
|
|
req = self.tts_requests.get(msg.get("id"))
|
|
if req is not None and self.worker.tts:
|
|
self.worker.tts.cancel(req)
|
|
|
|
# --- history and brain ----------------------------------------------------------------------
|
|
def _strings(self, msg: dict, *keys: str) -> tuple | None:
|
|
values = tuple(msg.get(k) for k in keys)
|
|
if not all(isinstance(v, str) and v for v in values):
|
|
self.error("bad_message", f"{msg.get('type')} needs non-empty strings: {', '.join(keys)}",
|
|
**({"id": msg["id"]} if isinstance(msg.get("id"), str) else {}))
|
|
return None
|
|
return values
|
|
|
|
def _leave_voice(self) -> None:
|
|
if self.voice_channel and self.worker.history is not None:
|
|
self.worker.history.presence_close_channel(self.voice_channel, time.time())
|
|
if self.voice_channel:
|
|
debug_log(f"session {self.sid} left voice channel {self.voice_channel}", "voice_worker")
|
|
self.voice_channel = None
|
|
|
|
def _on_voice_join(self, msg: dict) -> None:
|
|
got = self._strings(msg, "channel")
|
|
if got is None:
|
|
return
|
|
self._leave_voice()
|
|
self.voice_channel = got[0]
|
|
debug_log(f"session {self.sid} in voice channel {self.voice_channel}", "voice_worker")
|
|
|
|
def _on_member_join(self, msg: dict) -> None:
|
|
got = self._strings(msg, "user", "name")
|
|
if got is None:
|
|
return
|
|
user, name = got
|
|
self.names[user] = name
|
|
if self.voice_channel and self.worker.history is not None:
|
|
self.worker.history.presence_join(user, self.voice_channel, time.time())
|
|
|
|
def _on_member_leave(self, msg: dict) -> None:
|
|
got = self._strings(msg, "user")
|
|
if got is not None and self.voice_channel and self.worker.history is not None:
|
|
self.worker.history.presence_leave(got[0], self.voice_channel, time.time())
|
|
|
|
def _on_chat_message(self, msg: dict) -> None:
|
|
got = self._strings(msg, "channel", "message_id", "author", "name")
|
|
ts, to, text = msg.get("ts"), msg.get("to"), msg.get("text")
|
|
if got is None:
|
|
return
|
|
if not isinstance(text, str) or isinstance(ts, bool) or not isinstance(ts, (int, float)) \
|
|
or (to is not None and not isinstance(to, str)):
|
|
self.error("bad_message", "chat_message needs text, a numeric ts and an optional string to")
|
|
return
|
|
if self.worker.history is not None:
|
|
channel, message_id, author, name = got
|
|
self.worker.history.add_chat(channel, message_id, author, name, text, float(ts), to_id=to)
|
|
|
|
def _on_chat_edit(self, msg: dict) -> None:
|
|
got = self._strings(msg, "channel", "message_id")
|
|
if got is None:
|
|
return
|
|
if not isinstance(msg.get("text"), str):
|
|
self.error("bad_message", "chat_edit needs text")
|
|
return
|
|
if self.worker.history is not None:
|
|
self.worker.history.edit_chat(got[0], got[1], msg["text"])
|
|
|
|
def _on_chat_delete(self, msg: dict) -> None:
|
|
got = self._strings(msg, "channel", "message_id")
|
|
if got is not None and self.worker.history is not None:
|
|
self.worker.history.delete_chat(*got)
|
|
|
|
_CHAT_FLAGS = ("read", "send", "edit", "react", "delete", "pin")
|
|
|
|
def _on_chat_channels(self, msg: dict) -> None:
|
|
channels = msg.get("channels")
|
|
ok = isinstance(channels, list) and all(
|
|
isinstance(c, dict) and all(isinstance(c.get(k), str) for k in ("id", "name", "description"))
|
|
and c["id"] and all(isinstance(c.get(f), bool) for f in self._CHAT_FLAGS) for c in channels)
|
|
if not ok:
|
|
self.error("bad_message", "chat_channels needs channels with id, name, description and "
|
|
"boolean read, send, edit, react, delete, pin")
|
|
return
|
|
self.chat_channels = [{k: c[k] for k in ("id", "name", "description", *self._CHAT_FLAGS)} for c in channels]
|
|
debug_log(f"session {self.sid} chat channels: {len(self.chat_channels)}", "voice_worker")
|
|
|
|
async def chat_action(self, action: str, channel: str, args: dict) -> dict:
|
|
"""Ask the app to do a chat action and wait for its answer (or a timeout)."""
|
|
aid = f"{self.worker.chat_action_prefix}-{self.sid}-{next(self._chat_ids)}"
|
|
fut = asyncio.get_running_loop().create_future()
|
|
self.chat_waiting[aid] = fut
|
|
self.send({"type": "chat_action", "id": aid, "action": action, "channel": channel,
|
|
**{k: v for k, v in args.items() if v is not None}})
|
|
debug_log(f"chat action {aid} {action} channel={channel}", "voice_worker")
|
|
try:
|
|
return await asyncio.wait_for(fut, self.worker.chat_timeout_s)
|
|
except asyncio.TimeoutError:
|
|
return {"ok": False, "error": "the app did not answer in time"}
|
|
finally:
|
|
self.chat_waiting.pop(aid, None)
|
|
|
|
def _on_chat_action_result(self, msg: dict) -> None:
|
|
fut = self.chat_waiting.get(msg.get("id"))
|
|
if fut is None or fut.done():
|
|
return
|
|
ids = msg.get("message_ids") or []
|
|
result = {"ok": msg.get("ok") is True}
|
|
if result["ok"]:
|
|
result["message_ids"] = [str(i) for i in ids if isinstance(i, (str, int))]
|
|
else:
|
|
result["error"] = str(msg.get("error") or "the app could not do it")
|
|
fut.set_result(result)
|
|
|
|
async def app_action(self, kind: str, action: str, args: dict) -> dict:
|
|
"""Ask the app to act on a schedule, a note or the screen (``kind``) and wait for its answer
|
|
(or a timeout)."""
|
|
aid = f"{self.worker.chat_action_prefix}-{self.sid}-{next(self._chat_ids)}"
|
|
fut = asyncio.get_running_loop().create_future()
|
|
self.chat_waiting[aid] = fut
|
|
self.send({**args, "type": f"{kind}_action", "id": aid, "action": action})
|
|
debug_log(f"{kind} action {aid} {action}", "voice_worker")
|
|
try:
|
|
return await asyncio.wait_for(fut, self.worker.chat_timeout_s)
|
|
except asyncio.TimeoutError:
|
|
return {"ok": False, "error": "the app did not answer in time"}
|
|
finally:
|
|
self.chat_waiting.pop(aid, None)
|
|
|
|
def _on_app_action_result(self, msg: dict) -> None:
|
|
fut = self.chat_waiting.get(msg.get("id"))
|
|
if fut is None or fut.done():
|
|
return
|
|
if msg.get("ok") is not True:
|
|
fut.set_result({"ok": False, "error": str(msg.get("error") or "the app could not do it")})
|
|
return
|
|
result = {"ok": True}
|
|
if isinstance(msg.get("item"), dict):
|
|
result["item"] = msg["item"]
|
|
if isinstance(msg.get("items"), list):
|
|
result["items"] = [i for i in msg["items"] if isinstance(i, dict)]
|
|
if isinstance(msg.get("images"), list): # screen pictures, base64 JPEG
|
|
result["images"] = [i for i in msg["images"] if isinstance(i, str)]
|
|
fut.set_result(result)
|
|
|
|
def _on_memory_config(self, msg: dict) -> None:
|
|
changes = {k: v for k, v in msg.items() if k != "type"}
|
|
try:
|
|
self.memory.update(changes)
|
|
except ValueError as e:
|
|
self.error("bad_message", str(e))
|
|
return
|
|
debug_log(f"session {self.sid} memory settings changed: {sorted(changes)}", "voice_worker")
|
|
|
|
def _on_reply(self, msg: dict) -> None:
|
|
got = self._strings(msg, "id", "user", "text")
|
|
if got is None:
|
|
return
|
|
rid, user, text = got
|
|
via, speak, channel = msg.get("via"), msg.get("speak", False), msg.get("channel")
|
|
notes = msg.get("notes", [])
|
|
if via not in ("voice", "chat", "task") or not isinstance(speak, bool) or len(rid.encode()) > 255 \
|
|
or (via == "task" and speak) or not (channel is None or isinstance(channel, str)) \
|
|
or not (isinstance(notes, list) and len(notes) <= MAX_NOTES
|
|
and all(isinstance(n, str) and len(n) <= MAX_NOTE_CHARS for n in notes)):
|
|
self.error("bad_message", "reply needs via voice|chat|task, a boolean speak (false for a task), "
|
|
"an optional string channel, at most 20 string notes and an id under 256 bytes", id=rid)
|
|
return
|
|
if self.worker.brain is None:
|
|
self.error("brain_unavailable", "the worker was started without --brain", id=rid)
|
|
return
|
|
if via == "task":
|
|
asyncio.get_running_loop().create_task(self._task(rid, user, text, notes))
|
|
return
|
|
if speak:
|
|
self.thinking_replies.add(rid)
|
|
if user in self.preparing:
|
|
self.preparing[user] = None # the call was answered: no newer text to prepare
|
|
self.last_prepared.pop(user, None)
|
|
waiting = self.worker.replies_unanswered
|
|
early = waiting.get(user, 0) > 0 # sent before their previous answer: not a reply to it
|
|
waiting[user] = waiting.get(user, 0) + 1
|
|
asyncio.get_running_loop().create_task(self._reply(rid, user, text, via, speak, early, channel, notes))
|
|
|
|
@staticmethod
|
|
def _records_step(history, notes: list | None) -> dict:
|
|
"""The first step of an answer's trace: the records and notes it was given."""
|
|
return {"kind": "records", "recent": list(history.recent), "matches": list(history.matches),
|
|
"notes": len(notes or [])}
|
|
|
|
async def _task(self, rid: str, user: str, text: str, notes: list | None = None) -> None:
|
|
"""A scheduled task (schedule.spec.md, "Tasks"): answered in the chat lane with the read-only
|
|
tools, its own short-term memory and the person's history scope. It is not a turn of their
|
|
conversation, so their turn count and delete confirmations stay as they are."""
|
|
trace = _LiveTrace(self.worker.history, "task", None, user, text)
|
|
try:
|
|
await self._task_inner(rid, user, text, notes, trace)
|
|
finally:
|
|
trace.finish("") # only when it did not get to its answer
|
|
|
|
async def _task_inner(self, rid: str, user: str, text: str, notes: list | None, trace: _LiveTrace) -> None:
|
|
w = self.worker
|
|
answer = None
|
|
async with w.brain_people.setdefault(user, asyncio.Lock()), w.brain_lanes["chat"]:
|
|
if self.closed:
|
|
return
|
|
now = time.time()
|
|
if w.history is not None:
|
|
m = self.memory
|
|
history = await asyncio.to_thread(
|
|
w.history.context, user, text, now, m.scope(now), minutes=m.recent_minutes,
|
|
max_records=m.recent_records, limit=m.matches, budget_tokens=m.budget_tokens)
|
|
else:
|
|
from jarvis.memory.history import HistoryContext
|
|
history = HistoryContext()
|
|
try:
|
|
trace.append(self._records_step(history, notes))
|
|
async with self._llm():
|
|
answer = await asyncio.to_thread(w.brain.reply, user, text, history, task=True,
|
|
notes=notes or [], trace=trace)
|
|
except Exception as e: # the brain must never take the session down
|
|
debug_log(f"brain task {rid} failed: {type(e).__name__}", "llm")
|
|
answer = (answer or "").strip()
|
|
trace.finish(answer)
|
|
self.send({"type": "reply_text", "id": rid, "text": answer, "failed": not answer})
|
|
|
|
def _on_llm(self, msg: dict) -> None:
|
|
state, notice, sid, game = msg.get("state"), msg.get("notice"), msg.get("id"), msg.get("game", False)
|
|
if state not in ("switching", "ready", "paused") or not (notice is None or isinstance(notice, str)) \
|
|
or not (sid is None or (isinstance(sid, int) and not isinstance(sid, bool))) or not isinstance(game, bool):
|
|
self.error("bad_message", "llm needs state switching|ready|paused, an optional string notice, integer id and boolean game")
|
|
return
|
|
w = self.worker
|
|
w.llm_state = state
|
|
if state == "ready":
|
|
w.llm_game = game
|
|
if w.scheduler:
|
|
w.scheduler.wake()
|
|
debug_log(f"llm {state}", "llm")
|
|
if state == "ready":
|
|
w.llm_notice, w.llm_idle_to = None, None
|
|
w.llm_ready.set()
|
|
return
|
|
w.llm_ready.clear()
|
|
if state == "switching":
|
|
w.llm_notice, w.llm_noticed = notice or None, False
|
|
w.llm_idle_to, w.llm_switch_id = self, sid
|
|
self._llm_check_idle()
|
|
else:
|
|
w.llm_ready.set() # paused: waiting requests stop waiting and answer nothing
|
|
|
|
def _llm_check_idle(self) -> None:
|
|
w = self.worker
|
|
if w.llm_state == "switching" and w.llm_active == 0 and w.llm_idle_to is not None:
|
|
to, w.llm_idle_to = w.llm_idle_to, None
|
|
to.send({"type": "llm_idle"} if w.llm_switch_id is None else {"type": "llm_idle", "id": w.llm_switch_id})
|
|
|
|
@contextlib.asynccontextmanager
|
|
async def _llm(self, notice_id: str | None = None):
|
|
"""Brain work that uses the LLM: it waits while the LLM switches (the first spoken request
|
|
that waits hears the notice once, under its own id), fails while paused or after waiting
|
|
`LLM_WAIT_S`, and counts as running so a switch waits for it."""
|
|
w = self.worker
|
|
while w.llm_state != "ready":
|
|
if w.llm_state == "paused":
|
|
raise _LlmPaused()
|
|
if notice_id and w.llm_notice and not w.llm_noticed:
|
|
w.llm_noticed = True
|
|
# The newline ends it as a sentence, so it is spoken now, not joined to the answer.
|
|
self._on_tts({"type": "tts", "id": notice_id, "text": w.llm_notice + "\n", "more": True})
|
|
try:
|
|
await asyncio.wait_for(w.llm_ready.wait(), LLM_WAIT_S)
|
|
except asyncio.TimeoutError:
|
|
# The LLM is still away (or a measuring server stands in): fail, never call it.
|
|
raise _LlmPaused() from None
|
|
w.llm_active += 1
|
|
try:
|
|
yield
|
|
finally:
|
|
w.llm_active -= 1
|
|
self._llm_check_idle()
|
|
|
|
def _on_prepare(self, msg: dict) -> None:
|
|
"""A call still being said: the brain gets ready for it. One prepare per user runs at a time;
|
|
while it runs only the newest text waits, and one about the same as the last one prepared
|
|
(or being prepared) is dropped. Nothing is sent back."""
|
|
from .brain import similar
|
|
got = self._strings(msg, "user", "text")
|
|
if got is None:
|
|
return
|
|
user, text = got
|
|
if self.worker.brain is None or not hasattr(self.worker.brain, "prepare"):
|
|
return
|
|
if user in self.last_prepared and similar(self.last_prepared[user], text):
|
|
return
|
|
if user in self.preparing:
|
|
self.preparing[user] = text
|
|
return
|
|
self.preparing[user] = None
|
|
asyncio.get_running_loop().create_task(self._prepare(user, text))
|
|
|
|
async def _prepare(self, user: str, text: str | None) -> None:
|
|
from .brain import similar
|
|
try:
|
|
while text is not None and not self.closed:
|
|
self.last_prepared[user] = text
|
|
try:
|
|
async with self._llm():
|
|
await asyncio.to_thread(self.worker.brain.prepare, user, text, bool(self.chat_channels))
|
|
except Exception as e: # the brain must never take the session down
|
|
debug_log(f"brain prepare failed: {type(e).__name__}", "llm")
|
|
newer, self.preparing[user] = self.preparing[user], None
|
|
text = newer if newer is not None and not similar(newer, self.last_prepared.get(user, "")) else None
|
|
finally:
|
|
self.preparing.pop(user, None)
|
|
|
|
def _on_addressed(self, msg: dict) -> None:
|
|
got = self._strings(msg, "id", "user", "text")
|
|
if got is None:
|
|
return
|
|
qid, user, text = got
|
|
if self.worker.brain is None:
|
|
self.error("brain_unavailable", "the worker was started without --brain", id=qid)
|
|
return
|
|
# Who is in the voice channel (people, and of them who may call), when the app says.
|
|
count = lambda k: msg[k] if isinstance(msg.get(k), int) and not isinstance(msg[k], bool) and msg[k] > 0 else None
|
|
people, callers = count("people"), count("callers")
|
|
|
|
async def judge():
|
|
try:
|
|
async with self._llm():
|
|
yes = await asyncio.to_thread(self.worker.brain.addressed, user, text, list(self.settings.hotwords),
|
|
people, callers)
|
|
except Exception as e: # the brain must never take the session down
|
|
debug_log(f"brain addressed {qid} failed: {type(e).__name__}", "llm")
|
|
yes = None
|
|
debug_log(f"brain addressed {qid} -> {yes}", "llm")
|
|
self.send({"type": "addressed_result", "id": qid, "addressed": bool(yes), "failed": yes is None})
|
|
asyncio.get_running_loop().create_task(judge())
|
|
|
|
def _on_rank(self, msg: dict) -> None:
|
|
qid, cands = msg.get("id"), msg.get("candidates")
|
|
ok = isinstance(qid, str) and isinstance(cands, list) and all(
|
|
isinstance(c, dict) and all(isinstance(c.get(k), str) for k in ("id", "user", "text")) for c in cands)
|
|
if not ok:
|
|
self.error("bad_message", "rank needs a string id and candidates of string id, user and text")
|
|
return
|
|
if self.worker.brain is None:
|
|
self.error("brain_unavailable", "the worker was started without --brain", id=qid)
|
|
return
|
|
cands = [{"id": c["id"], "user": c["user"], "text": c["text"]} for c in cands]
|
|
|
|
async def rank():
|
|
try:
|
|
async with self._llm():
|
|
order = await asyncio.to_thread(self.worker.brain.rank, cands)
|
|
except Exception as e:
|
|
debug_log(f"brain rank {qid} failed: {type(e).__name__}", "llm")
|
|
order = None
|
|
known = {c["id"] for c in cands}
|
|
order = [i for i in (order or []) if i in known] if order is not None else None
|
|
debug_log(f"brain rank {qid} candidates={len(cands)} ok={order is not None}", "llm")
|
|
self.send({"type": "rank_result", "id": qid, "order": order or [], "failed": order is None})
|
|
asyncio.get_running_loop().create_task(rank())
|
|
|
|
async def _reply(self, rid: str, user: str, text: str, via: str, speak: bool, early: bool = False,
|
|
channel: str | None = None, notes: list | None = None) -> None:
|
|
trace = _LiveTrace(self.worker.history, via, self.voice_channel if via == "voice" else channel, user, text)
|
|
try:
|
|
await self._reply_inner(rid, user, text, via, speak, early, channel, notes or [], trace=trace)
|
|
finally:
|
|
trace.finish("") # only when it did not get to its answer
|
|
self.thinking_replies.discard(rid)
|
|
self.cancelled_replies.discard(rid)
|
|
if not self._handed_over.pop(rid, False): # else _delivered counts it
|
|
self._reached(user)
|
|
|
|
def _reached(self, user: str) -> None:
|
|
waiting = self.worker.replies_unanswered
|
|
waiting[user] -= 1
|
|
if not waiting[user]:
|
|
del waiting[user]
|
|
|
|
def _delivered(self, rid: str, ok: bool) -> None:
|
|
"""The answer ``rid`` reached its asker (spoken, or posted by the app), or never will."""
|
|
got = self.deliveries.pop(rid, None)
|
|
if got is None:
|
|
return
|
|
user, turn, timer = got
|
|
if timer is not None:
|
|
timer.cancel()
|
|
if not ok:
|
|
# They never got this answer, so nothing it asked can be confirmed next.
|
|
w = self.worker
|
|
w.reply_turns[user] = max(w.reply_turns.get(user, 0), turn + 1)
|
|
debug_log(f"brain reply {rid} did not reach the asker", "llm")
|
|
self._reached(user)
|
|
|
|
def _on_reply_delivered(self, msg: dict) -> None:
|
|
rid, ok = msg.get("id"), msg.get("ok")
|
|
if not isinstance(rid, str) or not isinstance(ok, bool):
|
|
self.error("bad_message", "reply_delivered needs a string id and a boolean ok")
|
|
return
|
|
got = self.deliveries.get(rid)
|
|
if got is not None and got[2] is not None: # only chat answers wait for the app's report
|
|
self._delivered(rid, ok)
|
|
|
|
async def _reply_inner(self, rid: str, user: str, text: str, via: str, speak: bool, early: bool,
|
|
chat_channel: str | None = None, notes: list | None = None, *,
|
|
trace: _LiveTrace) -> None:
|
|
w = self.worker
|
|
channel = self.voice_channel
|
|
# Person first, then lane: a reply waiting for its asker's other answer holds no lane.
|
|
async with w.brain_people.setdefault(user, asyncio.Lock()), w.brain_lanes[via]:
|
|
if self.closed:
|
|
return
|
|
now = time.time()
|
|
if w.history is not None:
|
|
m = self.memory
|
|
history = await asyncio.to_thread(
|
|
w.history.context, user, text, now, m.scope(now), minutes=m.recent_minutes,
|
|
max_records=m.recent_records, limit=m.matches, budget_tokens=m.budget_tokens)
|
|
else:
|
|
from jarvis.memory.history import HistoryContext
|
|
history = HistoryContext()
|
|
turn = w.reply_turns[user] = w.reply_turns.get(user, 0) + (2 if early else 1)
|
|
if early:
|
|
debug_log(f"brain reply {rid} arrived before the previous answer: a turn is skipped",
|
|
"llm")
|
|
from .chat_bridge import ChatBridge
|
|
chat = ChatBridge(self, asyncio.get_running_loop(), turn, w.pending_deletes.setdefault(user, {}), user) \
|
|
if self.chat_channels else None
|
|
from .app_bridge import AppBridge
|
|
origin = (user, via, chat_channel if via == "chat" else None)
|
|
schedule = AppBridge(self, asyncio.get_running_loop(), "schedule", *origin)
|
|
notes_backend = AppBridge(self, asyncio.get_running_loop(), "note", *origin)
|
|
screen = AppBridge(self, asyncio.get_running_loop(), "screen", *origin)
|
|
trace.append(self._records_step(history, notes))
|
|
try:
|
|
async with self._llm(rid if speak else None):
|
|
w.answering += 1
|
|
try:
|
|
answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat,
|
|
schedule=schedule, notes=notes or [],
|
|
notes_backend=notes_backend, screen=screen,
|
|
trace=trace)
|
|
finally:
|
|
w.answering -= 1
|
|
if not w.answering and w.scheduler:
|
|
w.scheduler.wake() # held partials go on
|
|
except Exception as e: # the brain must never take the session down
|
|
debug_log(f"brain reply {rid} failed: {type(e).__name__}", "llm")
|
|
answer = None
|
|
answer = (answer or "").strip()
|
|
trace.finish(answer)
|
|
if not answer and rid in self.tts_requests:
|
|
self._on_tts({"type": "tts", "id": rid, "text": "", "more": False}) # ends the notice's speech
|
|
self.send({"type": "reply_text", "id": rid, "text": answer, "failed": not answer})
|
|
# The answer reaches them when spoken (tts_done) or when the app reports the post
|
|
# (reply_delivered); an unspoken voice answer has nothing more to wait for.
|
|
posted = via == "chat" and not speak
|
|
timer = asyncio.get_running_loop().call_later(w.delivery_timeout_s, self._delivered, rid, False) \
|
|
if posted else None
|
|
self.deliveries[rid] = (user, turn, timer)
|
|
self._handed_over[rid] = True
|
|
if self.closed or not answer or (speak and rid in self.cancelled_replies):
|
|
self._delivered(rid, False)
|
|
elif not speak and not posted:
|
|
self._delivered(rid, True)
|
|
if not answer:
|
|
return
|
|
if via == "voice" and channel and w.history is not None:
|
|
w.history.add_reply(channel, user, answer, time.time())
|
|
if speak and rid in self.cancelled_replies:
|
|
debug_log(f"brain reply {rid} not spoken: cancelled while answering", "llm")
|
|
self.send({"type": "tts_done", "id": rid, "chunks": 0, "cancelled": True, "failed": False})
|
|
elif speak:
|
|
self._on_tts({"type": "tts", "id": rid, "text": answer, "more": False})
|
|
|
|
def _tts_chunk_cb(self, rid: str):
|
|
def cb(seq: int, pcm: np.ndarray) -> None:
|
|
pcm = np.asarray(pcm, dtype="<i2")
|
|
rec = self.tts_recorded.get(rid)
|
|
if rec is not None:
|
|
rec["at"] = rec["at"] or time.time()
|
|
rec["pcm"].append(pcm.copy())
|
|
if self.tts_format["channels"] == 2:
|
|
pcm = np.repeat(pcm, 2)
|
|
self.send(encode_tts_frame(rid, seq, pcm.tobytes()))
|
|
return cb
|
|
|
|
def _tts_done(self, req: TtsRequest) -> None:
|
|
self.tts_requests.pop(req.id, None)
|
|
rec = self.tts_recorded.get(req.id)
|
|
if rec is not None:
|
|
rec["made"] = True
|
|
self._record_spoken(req.id)
|
|
self.send({"type": "tts_done", "id": req.id, "chunks": req.chunks, "cancelled": req.cancelled,
|
|
"failed": req.failed})
|
|
self._delivered(req.id, not (req.cancelled or req.failed))
|
|
|
|
def _on_tts_played(self, msg: dict) -> None:
|
|
"""How much of a spoken answer the app played: the recording keeps that much."""
|
|
rec = self.tts_recorded.get(msg.get("id"))
|
|
played = msg.get("played")
|
|
if rec is None or isinstance(played, bool) or not isinstance(played, int) or played < 0:
|
|
return
|
|
rec["played"], rec["cancelled"] = played, bool(msg.get("cancelled"))
|
|
self._record_spoken(msg["id"])
|
|
|
|
def _record_spoken(self, rid: str) -> None:
|
|
"""Keep a spoken answer once it is made and the app said how much of it was heard (whole
|
|
sentences, the last one maybe cut off by a stop)."""
|
|
rec = self.tts_recorded.get(rid)
|
|
if rec is None or not rec["made"] or rec["played"] is None:
|
|
return
|
|
self.tts_recorded.pop(rid)
|
|
heard = rec["pcm"][:rec["played"]]
|
|
if heard and self.worker.recorder is not None:
|
|
self.worker.recorder.spoke(rec["at"], rid, rec["text"], np.concatenate(heard), rec["rate"],
|
|
rec["cancelled"])
|
|
|
|
def _tts_error(self, req: TtsRequest, message: str) -> None:
|
|
self.error("tts_failed", message, id=req.id)
|