- Playback sends how long the clip playing when a request was stopped had played (tts_played last_ms); the worker cuts the last sentence of that recording to it. - botBrowser open takes the search words as plain `query`: the tool puts them, encoded, into the address's empty search parameter or types them into the site's search box. The description tells the model to search this way and never put words into an address itself (live: 아이유 hand-encoded into 아이우). Eval (qwen3:8b, 3 runs): right words reach YouTube 1/3 -> 3/3. - When a page's main video does not play (also after the tool pressed play, or it failed to load), a note after the page says it is not playing and not to say it plays. Eval: a paused video reported as playing 1/3 -> 0/3. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1002 lines
50 KiB
Python
1002 lines
50 KiB
Python
"""Local WebSocket server: one session per connection (protocol in voice_worker.spec.md)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import contextlib
|
|
import hmac
|
|
import ipaddress
|
|
import itertools
|
|
import json
|
|
import secrets
|
|
import time
|
|
from dataclasses import dataclass
|
|
from typing import Callable
|
|
|
|
import numpy as np
|
|
|
|
from .log import debug_log
|
|
|
|
from .audio import InputConverter, loudness_equalise
|
|
from .brain import Brain, MemorySettings
|
|
from .protocol import KIND_AUDIO, ProtocolError, decode_audio_frame, encode_tts_frame
|
|
from .scheduler import DecodeOptions, SttResult, SttScheduler, Transcriber
|
|
from .segmenter import SR, Final, Partial, Segmenter, SpeechStart, VadStream
|
|
from .settings import SttSettings
|
|
from .tts import TtsEngine, TtsRequest, TtsService
|
|
|
|
CLOSE_BAD_HELLO = 4400
|
|
CLOSE_BAD_TOKEN = 4401
|
|
MAX_SPEAKERS = 32
|
|
|
|
|
|
def is_loopback(host: str) -> bool:
|
|
if host == "localhost":
|
|
return True
|
|
try:
|
|
return ipaddress.ip_address(host).is_loopback
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
# How long a request waits for the LLM to come back from a switch (a restart takes 30-50 s; the
|
|
# first game switch also measures once).
|
|
LLM_WAIT_S = 180.0
|
|
|
|
|
|
class _LlmPaused(Exception):
|
|
"""The app paused the LLM (a game runs): nothing is answered until it is ready again."""
|
|
|
|
|
|
# The asker's standing instructions the app sends with a reply (personal_notes.spec.md).
|
|
MAX_NOTES = 20
|
|
MAX_NOTE_CHARS = 300
|
|
# How long answer traces stay (history.spec.md, "Traces"); removed when the worker starts.
|
|
TRACE_KEEP_DAYS = 30
|
|
|
|
|
|
class _LiveTrace(list):
|
|
"""A reply's trace (history.spec.md, "Traces"): stored as soon as the request arrives and again
|
|
as each step is added, so the 대화 screen fills it in while the answer is made. The engine adds
|
|
steps from the brain's thread. Storing never stops the answer."""
|
|
|
|
def __init__(self, store, via: str, channel: str | None, user: str, question: str):
|
|
super().__init__()
|
|
self.store, self.began, self.id = store, time.time(), None
|
|
if store is not None:
|
|
self.id = self._keep(store.start_trace, via, channel, user, question, ts=self.began)
|
|
|
|
@staticmethod
|
|
def _keep(write, *args, **kw):
|
|
try:
|
|
return write(*args, **kw)
|
|
except Exception as e: # noqa: BLE001 - a trace is a view, not part of the answer
|
|
debug_log(f"trace not kept: {type(e).__name__}", "llm")
|
|
return None
|
|
|
|
def append(self, step) -> None:
|
|
super().append(step)
|
|
if self.id is not None:
|
|
self._keep(self.store.trace_steps, self.id, list(self))
|
|
|
|
def finish(self, answer: str) -> None:
|
|
if self.id is not None:
|
|
self._keep(self.store.finish_trace, self.id, answer, list(self),
|
|
ms=round((time.time() - self.began) * 1000))
|
|
self.id = None
|
|
|
|
|
|
class VoiceWorker:
|
|
def _partials_held(self) -> bool:
|
|
return self.llm_game and self.answering > 0
|
|
|
|
def __init__(self, token: str, transcriber: Transcriber | None, vad_factory: Callable[[], VadStream],
|
|
tts_engine: TtsEngine | None = None, hello_timeout_s: float = 5.0, tick_s: float = 0.05,
|
|
history=None, brain: Brain | None = None, chat_timeout_s: float = 15.0,
|
|
delivery_timeout_s: float = 30.0, recorder=None):
|
|
if not token:
|
|
raise ValueError("a connection token is required")
|
|
self.token = token
|
|
self.transcriber = transcriber
|
|
self.vad_factory = vad_factory
|
|
self.tts_engine = tts_engine
|
|
self.hello_timeout_s = hello_timeout_s
|
|
self.tick_s = tick_s
|
|
self.history = history # jarvis.memory.history.HistoryStore or None
|
|
self.recorder = recorder # recordings.Recorder (voice.record on) or None
|
|
self.brain = brain
|
|
# One reply at a time per lane: voice answers never wait behind another person's chat answer
|
|
# (conversation.md: with an "answer everything" chat channel the LLM takes two or more
|
|
# requests at once). One person's replies stay one at a time across lanes, so their dialogue
|
|
# memory and delete confirmations ("the very next message") follow the order they asked.
|
|
self.brain_lanes = {"voice": asyncio.Lock(), "chat": asyncio.Lock()}
|
|
self.brain_people: dict[str, asyncio.Lock] = {}
|
|
# The LLM's state from the app (voice_worker.spec.md, "LLM switches"): while it switches or
|
|
# is paused no new brain work starts; the brain work running when a switch is announced
|
|
# finishes first, then the app is told the brain is idle.
|
|
self.llm_state = "ready"
|
|
self.llm_ready = asyncio.Event()
|
|
self.llm_ready.set()
|
|
self.llm_notice: str | None = None
|
|
self.llm_noticed = False
|
|
self.llm_active = 0
|
|
self.llm_idle_to = None # the session to tell once nothing runs
|
|
self.llm_switch_id = None # the switch to name in llm_idle
|
|
# In a game (the app says so with `ready`), partial passes wait while an answer is written
|
|
# (M0 gamemode_tune: the LLM answers 1.7 times faster); finals never wait.
|
|
self.llm_game = False
|
|
self.answering = 0
|
|
self.chat_timeout_s = chat_timeout_s
|
|
self.delivery_timeout_s = delivery_timeout_s # a chat answer the app never reports: not posted
|
|
# chat_action ids are unique across connections and worker runs, so a late answer from an
|
|
# earlier connection (or an earlier worker) never completes a newer action.
|
|
self.chat_action_prefix = secrets.token_hex(4)
|
|
# Per person: how many replies they had (a delete is confirmed only in the very next one),
|
|
# the deletes they were asked to confirm (chat_channels.spec.md) and how many of their
|
|
# replies have not reached them yet (spoken, or posted by the app). A turn number is skipped
|
|
# for a request that arrived before the person's previous answer reached them, and after an
|
|
# answer that never did, so a confirmation asked in that answer lapses.
|
|
self.reply_turns: dict[str, int] = {}
|
|
self.pending_deletes: dict[str, dict] = {}
|
|
self.replies_unanswered: dict[str, int] = {}
|
|
self.scheduler: SttScheduler | None = None
|
|
self.tts: TtsService | None = None
|
|
self._ids = itertools.count(1)
|
|
|
|
async def start(self) -> None:
|
|
if self.history is not None:
|
|
# A killed worker never closed its sessions' presence; nobody is heard until the app
|
|
# reports who is there again.
|
|
n = self.history.close_open_presence(time.time())
|
|
if n:
|
|
debug_log(f"closed {n} presence intervals left open by a previous run", "voice_worker")
|
|
n = self.history.close_unfinished_traces()
|
|
if n:
|
|
debug_log(f"closed {n} answer traces left unfinished by a previous run", "voice_worker")
|
|
n = self.history.purge_traces_older_than(time.time() - TRACE_KEEP_DAYS * 86400)
|
|
if n:
|
|
debug_log(f"removed {n} answer traces older than {TRACE_KEEP_DAYS} days", "voice_worker")
|
|
if self.transcriber is not None:
|
|
self.scheduler = SttScheduler(self.transcriber, partials_held=self._partials_held)
|
|
self.scheduler.start()
|
|
if self.tts_engine is not None:
|
|
self.tts = TtsService(self.tts_engine)
|
|
self.tts.start()
|
|
|
|
async def stop(self) -> None:
|
|
if self.scheduler:
|
|
await self.scheduler.stop()
|
|
if self.tts:
|
|
await self.tts.stop()
|
|
|
|
async def serve(self, host: str, port: int):
|
|
"""Start listening; returns the websockets server. Loopback addresses only."""
|
|
if not is_loopback(host):
|
|
raise ValueError("the voice worker only listens on a loopback address")
|
|
from websockets.asyncio.server import serve
|
|
await self.start()
|
|
return await serve(self.handle, host, port, max_size=4 * 2 ** 20) # pictures (screen_action_result)
|
|
|
|
async def handle(self, ws) -> None:
|
|
try:
|
|
first = await asyncio.wait_for(ws.recv(), self.hello_timeout_s)
|
|
hello = json.loads(first) if isinstance(first, str) else None
|
|
except (asyncio.TimeoutError, ValueError):
|
|
hello = None
|
|
except Exception:
|
|
return
|
|
if not isinstance(hello, dict) or hello.get("type") != "hello":
|
|
debug_log("connection refused: no hello", "voice_worker")
|
|
await ws.close(CLOSE_BAD_HELLO, "hello expected")
|
|
return
|
|
token = hello.get("token")
|
|
if not isinstance(token, str) or not hmac.compare_digest(token.encode(), self.token.encode()):
|
|
debug_log("connection refused: bad token", "voice_worker")
|
|
await ws.close(CLOSE_BAD_TOKEN, "bad token")
|
|
return
|
|
try:
|
|
session = Session(self, ws, next(self._ids), hello)
|
|
except ValueError as e:
|
|
await ws.close(CLOSE_BAD_HELLO, str(e)[:100])
|
|
return
|
|
await session.run()
|
|
|
|
|
|
@dataclass
|
|
class _Speaker:
|
|
converter: InputConverter
|
|
segmenter: Segmenter
|
|
|
|
|
|
def _format(value, default: dict, rates: tuple[int, ...] | None) -> dict:
|
|
if value is None:
|
|
return dict(default)
|
|
if not isinstance(value, dict):
|
|
raise ValueError("format must be an object")
|
|
rate, channels = value.get("rate", default["rate"]), value.get("channels", default["channels"])
|
|
ok_rate = isinstance(rate, int) and (rate in rates if rates else 8000 <= rate <= 96000)
|
|
if not ok_rate or channels not in (1, 2) or isinstance(channels, bool):
|
|
raise ValueError("unsupported audio format")
|
|
return {"rate": rate, "channels": channels}
|
|
|
|
|
|
class Session:
|
|
def __init__(self, worker: VoiceWorker, ws, sid: int, hello: dict):
|
|
self.worker = worker
|
|
self.ws = ws
|
|
self.sid = sid
|
|
self.audio = _format(hello.get("audio"), {"rate": 48000, "channels": 2}, (16000, 48000))
|
|
self.tts_format = _format(hello.get("tts"), {"rate": 48000, "channels": 2}, None)
|
|
self.settings = SttSettings()
|
|
stt = hello.get("stt")
|
|
if stt is not None:
|
|
if not isinstance(stt, dict):
|
|
raise ValueError("stt must be an object")
|
|
self.settings.update(stt)
|
|
self.memory = MemorySettings()
|
|
memory = hello.get("memory")
|
|
if memory is not None:
|
|
if not isinstance(memory, dict):
|
|
raise ValueError("memory must be an object")
|
|
self.memory.update(memory)
|
|
self.voice_channel: str | None = None
|
|
self.chat_channels: list[dict] = []
|
|
self.chat_waiting: dict[str, asyncio.Future] = {}
|
|
self._chat_ids = itertools.count(1)
|
|
self.names: dict[str, str] = {}
|
|
self.speakers: dict[str, _Speaker] = {}
|
|
self.tts_requests: dict[str, TtsRequest] = {}
|
|
# With recordings on: each spoken answer's text and mono samples, kept until it is made and the
|
|
# app said how much of it was played (`tts_played`).
|
|
self.tts_recorded: dict[str, dict] = {}
|
|
# Spoken replies whose speech was cancelled while the brain was still answering.
|
|
self.cancelled_replies: set[str] = set()
|
|
self.thinking_replies: set[str] = set()
|
|
# user -> the newest call text waiting for that user's running prepare to finish (or None).
|
|
self.preparing: dict[str, str | None] = {}
|
|
self.last_prepared: dict[str, str] = {} # user -> the text of their last prepare
|
|
# Answers sent but not yet known to have reached the person: id -> (user, turn, timer).
|
|
self.deliveries: dict[str, tuple[str, int, asyncio.TimerHandle | None]] = {}
|
|
self._handed_over: dict[str, bool] = {} # replies whose end is counted by _delivered
|
|
self.out: asyncio.Queue = asyncio.Queue()
|
|
self.closed = False
|
|
|
|
# --- outbound -------------------------------------------------------------------------------
|
|
def send(self, msg) -> None:
|
|
if not self.closed:
|
|
self.out.put_nowait(msg if isinstance(msg, bytes) else json.dumps(msg, ensure_ascii=False))
|
|
|
|
def error(self, code: str, message: str, **extra) -> None:
|
|
self.send({"type": "error", "code": code, "message": message, **extra})
|
|
|
|
async def _writer(self) -> None:
|
|
while True:
|
|
await self.ws.send(await self.out.get())
|
|
|
|
# --- lifecycle ------------------------------------------------------------------------------
|
|
async def run(self) -> None:
|
|
w = self.worker
|
|
debug_log(f"session {self.sid} connected audio={self.audio} tts={self.tts_format}", "voice_worker")
|
|
self.send({"type": "hello_ok", "stt_ready": w.scheduler is not None, "tts_ready": w.tts is not None,
|
|
"audio": self.audio, "tts": self.tts_format, "stt": self.settings.as_dict()})
|
|
writer = asyncio.create_task(self._writer())
|
|
ticker = asyncio.create_task(self._ticker())
|
|
try:
|
|
async for frame in self.ws:
|
|
if isinstance(frame, bytes):
|
|
self._on_binary(frame)
|
|
else:
|
|
self._on_text(frame)
|
|
except Exception as e: # connection dropped
|
|
debug_log(f"session {self.sid} receive ended: {type(e).__name__}", "voice_worker")
|
|
finally:
|
|
self.closed = True
|
|
self._leave_voice()
|
|
for fut in self.chat_waiting.values():
|
|
if not fut.done():
|
|
fut.set_result({"ok": False, "error": "the app disconnected"})
|
|
for rid in list(self.deliveries):
|
|
self._delivered(rid, False)
|
|
ticker.cancel()
|
|
writer.cancel()
|
|
if w.scheduler:
|
|
w.scheduler.drop_partials(lambda key: key[0] == self.sid)
|
|
if w.tts:
|
|
for req in list(self.tts_requests.values()):
|
|
w.tts.cancel(req)
|
|
debug_log(f"session {self.sid} closed", "voice_worker")
|
|
|
|
async def _ticker(self) -> None:
|
|
while True:
|
|
await asyncio.sleep(self.worker.tick_s)
|
|
now = time.monotonic()
|
|
for speaker, st in list(self.speakers.items()):
|
|
self._handle_events(speaker, st, st.segmenter.tick(now))
|
|
|
|
# --- STT ------------------------------------------------------------------------------------
|
|
def _on_binary(self, frame: bytes) -> None:
|
|
if not frame or frame[0] != KIND_AUDIO:
|
|
self.error("bad_frame", "unknown binary frame kind")
|
|
return
|
|
try:
|
|
speaker, pcm = decode_audio_frame(frame)
|
|
except ProtocolError as e:
|
|
self.error("bad_frame", str(e))
|
|
return
|
|
st = self.speakers.get(speaker)
|
|
if st is None:
|
|
if len(self.speakers) >= MAX_SPEAKERS:
|
|
self.error("bad_frame", f"more than {MAX_SPEAKERS} speakers")
|
|
return
|
|
st = _Speaker(InputConverter(self.audio["rate"], self.audio["channels"]),
|
|
Segmenter(self.worker.vad_factory(), self.settings))
|
|
self.speakers[speaker] = st
|
|
samples = st.converter.push(pcm)
|
|
self._handle_events(speaker, st, st.segmenter.push(samples, time.monotonic()))
|
|
|
|
def _opts(self, beam: int) -> DecodeOptions:
|
|
return DecodeOptions(beam=beam, language=self.settings.language, hotwords=self.settings.hotwords)
|
|
|
|
def _handle_events(self, speaker: str, st: _Speaker, events: list) -> None:
|
|
sched = self.worker.scheduler
|
|
for ev in events:
|
|
if isinstance(ev, SpeechStart):
|
|
self.send({"type": "speech_start", "speaker": speaker, "utt": ev.utt})
|
|
elif isinstance(ev, Partial):
|
|
if sched and self.settings.partials:
|
|
sched.submit_partial((self.sid, speaker), loudness_equalise(ev.audio, ev.voiced), self._opts(1),
|
|
self._partial_cb(speaker, st, ev.utt))
|
|
elif isinstance(ev, Final):
|
|
debug_log(f"utterance end speaker={speaker} utt={ev.utt} reason={ev.reason} "
|
|
f"ms={len(ev.audio) * 1000 // SR}", "stt")
|
|
# Where, when and by whom it was said, fixed now: decoding can take a while, and a
|
|
# move or a late joiner meanwhile must not change who heard it.
|
|
said = (self.voice_channel, self.names.get(speaker, speaker),
|
|
time.time() - (time.monotonic() - ev.decided_at))
|
|
if sched is None:
|
|
self._send_final(speaker, ev, SttResult("", 0, "stt unavailable"), said)
|
|
else:
|
|
sched.submit_final((self.sid, speaker), loudness_equalise(ev.audio, ev.voiced),
|
|
self._opts(self.settings.final_beam),
|
|
lambda res, ev=ev, said=said: self._send_final(speaker, ev, res, said))
|
|
|
|
def _partial_cb(self, speaker: str, st: _Speaker, utt: int):
|
|
def cb(res: SttResult) -> None:
|
|
# Drop it when the utterance already ended or captions were turned off meanwhile.
|
|
if st.segmenter.open_utt == utt and self.settings.partials and not res.error:
|
|
self.send({"type": "partial", "speaker": speaker, "utt": utt, "text": res.text})
|
|
return cb
|
|
|
|
def _send_final(self, speaker: str, ev: Final, res: SttResult, said: tuple) -> None:
|
|
channel, name, ts = said
|
|
store = self.worker.history
|
|
if store is not None and channel and res.text.strip():
|
|
store.add_voice(channel, speaker, name, res.text, ts)
|
|
if self.worker.recorder is not None:
|
|
self.worker.recorder.said(ts, speaker, name, channel, res.text, ev.audio, SR)
|
|
self.send({"type": "final", "speaker": speaker, "utt": ev.utt, "text": res.text,
|
|
"end_reason": ev.reason, "audio_ms": len(ev.audio) * 1000 // SR,
|
|
"wait_ms": int((time.monotonic() - ev.decided_at) * 1000), "stt_ms": res.stt_ms})
|
|
|
|
# --- text messages --------------------------------------------------------------------------
|
|
def _on_text(self, raw: str) -> None:
|
|
try:
|
|
msg = json.loads(raw)
|
|
except ValueError:
|
|
self.error("bad_message", "not JSON")
|
|
return
|
|
if not isinstance(msg, dict):
|
|
self.error("bad_message", "not an object")
|
|
return
|
|
kind = msg.get("type")
|
|
handler = {"speaking_end": self._on_speaking_end, "speaker_leave": self._on_speaker_leave,
|
|
"config": self._on_config, "tts": self._on_tts, "tts_cancel": self._on_tts_cancel,
|
|
"tts_speed": self._on_tts_speed, "tts_played": self._on_tts_played,
|
|
"voice_join": self._on_voice_join, "voice_leave": lambda m: self._leave_voice(),
|
|
"member_join": self._on_member_join, "member_leave": self._on_member_leave,
|
|
"chat_message": self._on_chat_message, "chat_edit": self._on_chat_edit,
|
|
"chat_delete": self._on_chat_delete, "reply": self._on_reply,
|
|
"addressed": self._on_addressed, "rank": self._on_rank,
|
|
"chat_channels": self._on_chat_channels, "chat_action_result": self._on_chat_action_result,
|
|
"schedule_action_result": self._on_app_action_result,
|
|
"note_action_result": self._on_app_action_result,
|
|
"screen_action_result": self._on_app_action_result,
|
|
"memory_config": self._on_memory_config, "reply_delivered": self._on_reply_delivered,
|
|
"prepare": self._on_prepare, "llm": self._on_llm}.get(kind)
|
|
if handler is None:
|
|
self.error("bad_message", f"unknown type: {kind!r}")
|
|
return
|
|
handler(msg)
|
|
|
|
def _on_speaking_end(self, msg: dict) -> None:
|
|
st = self.speakers.get(msg.get("speaker"))
|
|
if st is not None:
|
|
self._handle_events(msg["speaker"], st, st.segmenter.speaking_end(time.monotonic()))
|
|
|
|
def _on_speaker_leave(self, msg: dict) -> None:
|
|
# The speaker left the voice channel: finish any open utterance, then free their place.
|
|
speaker = msg.get("speaker")
|
|
st = self.speakers.pop(speaker, None) if isinstance(speaker, str) else None
|
|
if st is not None:
|
|
self._handle_events(speaker, st, st.segmenter.speaking_end(time.monotonic()))
|
|
if self.worker.scheduler:
|
|
self.worker.scheduler.drop_partials(lambda key: key == (self.sid, speaker))
|
|
debug_log(f"speaker {speaker} left session {self.sid}", "voice_worker")
|
|
|
|
def _on_config(self, msg: dict) -> None:
|
|
changes = {k: v for k, v in msg.items() if k != "type"}
|
|
try:
|
|
self.settings.update(changes)
|
|
except ValueError as e:
|
|
self.error("bad_message", str(e))
|
|
return
|
|
if not self.settings.partials and self.worker.scheduler:
|
|
self.worker.scheduler.drop_partials(lambda key: key[0] == self.sid)
|
|
debug_log(f"session {self.sid} settings changed: {sorted(changes)}", "voice_worker")
|
|
|
|
def _on_tts_speed(self, msg: dict) -> None:
|
|
speed = msg.get("speed")
|
|
if isinstance(speed, bool) or not isinstance(speed, (int, float)) or not 0.5 <= speed <= 2.0:
|
|
self.error("bad_message", "tts_speed needs a speed from 0.5 to 2.0")
|
|
return
|
|
if self.worker.tts_engine is not None:
|
|
self.worker.tts_engine.speed = float(speed)
|
|
debug_log(f"tts speed {speed}", "tts")
|
|
|
|
def _on_tts(self, msg: dict) -> None:
|
|
rid, text, more = msg.get("id"), msg.get("text", ""), msg.get("more", False)
|
|
if not isinstance(rid, str) or not 0 < len(rid.encode()) < 256 or not isinstance(text, str) \
|
|
or not isinstance(more, bool):
|
|
self.error("bad_message", "tts needs a string id and text, and a boolean more")
|
|
return
|
|
tts = self.worker.tts
|
|
if tts is None:
|
|
self.error("tts_unavailable", "no TTS engine is loaded", id=rid)
|
|
self._delivered(rid, False)
|
|
return
|
|
req = self.tts_requests.get(rid)
|
|
if req is None:
|
|
req = TtsRequest(rid, self.tts_format["rate"], self._tts_chunk_cb(rid), self._tts_done,
|
|
self._tts_error)
|
|
self.tts_requests[rid] = req
|
|
elif req.closed:
|
|
self.error("bad_message", "this tts id is already closed", id=rid)
|
|
return
|
|
if self.worker.recorder is not None:
|
|
rec = self.tts_recorded.setdefault(rid, {"text": "", "pcm": [], "at": None, "rate": req.rate,
|
|
"made": False, "played": None})
|
|
rec["text"] += text
|
|
while len(self.tts_recorded) > 50: # never reported as played: not kept forever
|
|
self.tts_recorded.pop(next(iter(self.tts_recorded)))
|
|
tts.feed(req, text, more)
|
|
|
|
def _on_tts_cancel(self, msg: dict) -> None:
|
|
if msg.get("id") in self.thinking_replies:
|
|
self.cancelled_replies.add(msg["id"])
|
|
req = self.tts_requests.get(msg.get("id"))
|
|
if req is not None and self.worker.tts:
|
|
self.worker.tts.cancel(req)
|
|
|
|
# --- history and brain ----------------------------------------------------------------------
|
|
def _strings(self, msg: dict, *keys: str) -> tuple | None:
|
|
values = tuple(msg.get(k) for k in keys)
|
|
if not all(isinstance(v, str) and v for v in values):
|
|
self.error("bad_message", f"{msg.get('type')} needs non-empty strings: {', '.join(keys)}",
|
|
**({"id": msg["id"]} if isinstance(msg.get("id"), str) else {}))
|
|
return None
|
|
return values
|
|
|
|
def _leave_voice(self) -> None:
|
|
if self.voice_channel and self.worker.history is not None:
|
|
self.worker.history.presence_close_channel(self.voice_channel, time.time())
|
|
if self.voice_channel:
|
|
debug_log(f"session {self.sid} left voice channel {self.voice_channel}", "voice_worker")
|
|
self.voice_channel = None
|
|
|
|
def _on_voice_join(self, msg: dict) -> None:
|
|
got = self._strings(msg, "channel")
|
|
if got is None:
|
|
return
|
|
self._leave_voice()
|
|
self.voice_channel = got[0]
|
|
debug_log(f"session {self.sid} in voice channel {self.voice_channel}", "voice_worker")
|
|
|
|
def _on_member_join(self, msg: dict) -> None:
|
|
got = self._strings(msg, "user", "name")
|
|
if got is None:
|
|
return
|
|
user, name = got
|
|
self.names[user] = name
|
|
if self.voice_channel and self.worker.history is not None:
|
|
self.worker.history.presence_join(user, self.voice_channel, time.time())
|
|
|
|
def _on_member_leave(self, msg: dict) -> None:
|
|
got = self._strings(msg, "user")
|
|
if got is not None and self.voice_channel and self.worker.history is not None:
|
|
self.worker.history.presence_leave(got[0], self.voice_channel, time.time())
|
|
|
|
def _on_chat_message(self, msg: dict) -> None:
|
|
got = self._strings(msg, "channel", "message_id", "author", "name")
|
|
ts, to, text = msg.get("ts"), msg.get("to"), msg.get("text")
|
|
if got is None:
|
|
return
|
|
if not isinstance(text, str) or isinstance(ts, bool) or not isinstance(ts, (int, float)) \
|
|
or (to is not None and not isinstance(to, str)):
|
|
self.error("bad_message", "chat_message needs text, a numeric ts and an optional string to")
|
|
return
|
|
if self.worker.history is not None:
|
|
channel, message_id, author, name = got
|
|
self.worker.history.add_chat(channel, message_id, author, name, text, float(ts), to_id=to)
|
|
|
|
def _on_chat_edit(self, msg: dict) -> None:
|
|
got = self._strings(msg, "channel", "message_id")
|
|
if got is None:
|
|
return
|
|
if not isinstance(msg.get("text"), str):
|
|
self.error("bad_message", "chat_edit needs text")
|
|
return
|
|
if self.worker.history is not None:
|
|
self.worker.history.edit_chat(got[0], got[1], msg["text"])
|
|
|
|
def _on_chat_delete(self, msg: dict) -> None:
|
|
got = self._strings(msg, "channel", "message_id")
|
|
if got is not None and self.worker.history is not None:
|
|
self.worker.history.delete_chat(*got)
|
|
|
|
_CHAT_FLAGS = ("read", "send", "edit", "react", "delete", "pin")
|
|
|
|
def _on_chat_channels(self, msg: dict) -> None:
|
|
channels = msg.get("channels")
|
|
ok = isinstance(channels, list) and all(
|
|
isinstance(c, dict) and all(isinstance(c.get(k), str) for k in ("id", "name", "description"))
|
|
and c["id"] and all(isinstance(c.get(f), bool) for f in self._CHAT_FLAGS) for c in channels)
|
|
if not ok:
|
|
self.error("bad_message", "chat_channels needs channels with id, name, description and "
|
|
"boolean read, send, edit, react, delete, pin")
|
|
return
|
|
self.chat_channels = [{k: c[k] for k in ("id", "name", "description", *self._CHAT_FLAGS)} for c in channels]
|
|
debug_log(f"session {self.sid} chat channels: {len(self.chat_channels)}", "voice_worker")
|
|
|
|
async def chat_action(self, action: str, channel: str, args: dict) -> dict:
|
|
"""Ask the app to do a chat action and wait for its answer (or a timeout)."""
|
|
aid = f"{self.worker.chat_action_prefix}-{self.sid}-{next(self._chat_ids)}"
|
|
fut = asyncio.get_running_loop().create_future()
|
|
self.chat_waiting[aid] = fut
|
|
self.send({"type": "chat_action", "id": aid, "action": action, "channel": channel,
|
|
**{k: v for k, v in args.items() if v is not None}})
|
|
debug_log(f"chat action {aid} {action} channel={channel}", "voice_worker")
|
|
try:
|
|
return await asyncio.wait_for(fut, self.worker.chat_timeout_s)
|
|
except asyncio.TimeoutError:
|
|
return {"ok": False, "error": "the app did not answer in time"}
|
|
finally:
|
|
self.chat_waiting.pop(aid, None)
|
|
|
|
def _on_chat_action_result(self, msg: dict) -> None:
|
|
fut = self.chat_waiting.get(msg.get("id"))
|
|
if fut is None or fut.done():
|
|
return
|
|
ids = msg.get("message_ids") or []
|
|
result = {"ok": msg.get("ok") is True}
|
|
if result["ok"]:
|
|
result["message_ids"] = [str(i) for i in ids if isinstance(i, (str, int))]
|
|
else:
|
|
result["error"] = str(msg.get("error") or "the app could not do it")
|
|
fut.set_result(result)
|
|
|
|
async def app_action(self, kind: str, action: str, args: dict) -> dict:
|
|
"""Ask the app to act on a schedule, a note or the screen (``kind``) and wait for its answer
|
|
(or a timeout)."""
|
|
aid = f"{self.worker.chat_action_prefix}-{self.sid}-{next(self._chat_ids)}"
|
|
fut = asyncio.get_running_loop().create_future()
|
|
self.chat_waiting[aid] = fut
|
|
self.send({**args, "type": f"{kind}_action", "id": aid, "action": action})
|
|
debug_log(f"{kind} action {aid} {action}", "voice_worker")
|
|
try:
|
|
return await asyncio.wait_for(fut, self.worker.chat_timeout_s)
|
|
except asyncio.TimeoutError:
|
|
return {"ok": False, "error": "the app did not answer in time"}
|
|
finally:
|
|
self.chat_waiting.pop(aid, None)
|
|
|
|
def _on_app_action_result(self, msg: dict) -> None:
|
|
fut = self.chat_waiting.get(msg.get("id"))
|
|
if fut is None or fut.done():
|
|
return
|
|
if msg.get("ok") is not True:
|
|
fut.set_result({"ok": False, "error": str(msg.get("error") or "the app could not do it")})
|
|
return
|
|
result = {"ok": True}
|
|
if isinstance(msg.get("item"), dict):
|
|
result["item"] = msg["item"]
|
|
if isinstance(msg.get("items"), list):
|
|
result["items"] = [i for i in msg["items"] if isinstance(i, dict)]
|
|
if isinstance(msg.get("images"), list): # screen pictures, base64 JPEG
|
|
result["images"] = [i for i in msg["images"] if isinstance(i, str)]
|
|
fut.set_result(result)
|
|
|
|
def _on_memory_config(self, msg: dict) -> None:
|
|
changes = {k: v for k, v in msg.items() if k != "type"}
|
|
try:
|
|
self.memory.update(changes)
|
|
except ValueError as e:
|
|
self.error("bad_message", str(e))
|
|
return
|
|
debug_log(f"session {self.sid} memory settings changed: {sorted(changes)}", "voice_worker")
|
|
|
|
def _on_reply(self, msg: dict) -> None:
|
|
got = self._strings(msg, "id", "user", "text")
|
|
if got is None:
|
|
return
|
|
rid, user, text = got
|
|
via, speak, channel = msg.get("via"), msg.get("speak", False), msg.get("channel")
|
|
notes = msg.get("notes", [])
|
|
if via not in ("voice", "chat", "task") or not isinstance(speak, bool) or len(rid.encode()) > 255 \
|
|
or (via == "task" and speak) or not (channel is None or isinstance(channel, str)) \
|
|
or not (isinstance(notes, list) and len(notes) <= MAX_NOTES
|
|
and all(isinstance(n, str) and len(n) <= MAX_NOTE_CHARS for n in notes)):
|
|
self.error("bad_message", "reply needs via voice|chat|task, a boolean speak (false for a task), "
|
|
"an optional string channel, at most 20 string notes and an id under 256 bytes", id=rid)
|
|
return
|
|
if self.worker.brain is None:
|
|
self.error("brain_unavailable", "the worker was started without --brain", id=rid)
|
|
return
|
|
if via == "task":
|
|
asyncio.get_running_loop().create_task(self._task(rid, user, text, notes))
|
|
return
|
|
if speak:
|
|
self.thinking_replies.add(rid)
|
|
if user in self.preparing:
|
|
self.preparing[user] = None # the call was answered: no newer text to prepare
|
|
self.last_prepared.pop(user, None)
|
|
waiting = self.worker.replies_unanswered
|
|
early = waiting.get(user, 0) > 0 # sent before their previous answer: not a reply to it
|
|
waiting[user] = waiting.get(user, 0) + 1
|
|
asyncio.get_running_loop().create_task(self._reply(rid, user, text, via, speak, early, channel, notes))
|
|
|
|
@staticmethod
|
|
def _records_step(history, notes: list | None) -> dict:
|
|
"""The first step of an answer's trace: the records and notes it was given."""
|
|
return {"kind": "records", "recent": list(history.recent), "matches": list(history.matches),
|
|
"notes": len(notes or [])}
|
|
|
|
async def _task(self, rid: str, user: str, text: str, notes: list | None = None) -> None:
|
|
"""A scheduled task (schedule.spec.md, "Tasks"): answered in the chat lane with the read-only
|
|
tools, its own short-term memory and the person's history scope. It is not a turn of their
|
|
conversation, so their turn count and delete confirmations stay as they are."""
|
|
trace = _LiveTrace(self.worker.history, "task", None, user, text)
|
|
try:
|
|
await self._task_inner(rid, user, text, notes, trace)
|
|
finally:
|
|
trace.finish("") # only when it did not get to its answer
|
|
|
|
async def _task_inner(self, rid: str, user: str, text: str, notes: list | None, trace: _LiveTrace) -> None:
|
|
w = self.worker
|
|
answer = None
|
|
async with w.brain_people.setdefault(user, asyncio.Lock()), w.brain_lanes["chat"]:
|
|
if self.closed:
|
|
return
|
|
now = time.time()
|
|
if w.history is not None:
|
|
m = self.memory
|
|
history = await asyncio.to_thread(
|
|
w.history.context, user, text, now, m.scope(now), minutes=m.recent_minutes,
|
|
max_records=m.recent_records, limit=m.matches, budget_tokens=m.budget_tokens)
|
|
else:
|
|
from jarvis.memory.history import HistoryContext
|
|
history = HistoryContext()
|
|
try:
|
|
trace.append(self._records_step(history, notes))
|
|
async with self._llm():
|
|
answer = await asyncio.to_thread(w.brain.reply, user, text, history, task=True,
|
|
notes=notes or [], trace=trace)
|
|
except Exception as e: # the brain must never take the session down
|
|
debug_log(f"brain task {rid} failed: {type(e).__name__}", "llm")
|
|
answer = (answer or "").strip()
|
|
trace.finish(answer)
|
|
self.send({"type": "reply_text", "id": rid, "text": answer, "failed": not answer})
|
|
|
|
def _on_llm(self, msg: dict) -> None:
|
|
state, notice, sid, game = msg.get("state"), msg.get("notice"), msg.get("id"), msg.get("game", False)
|
|
if state not in ("switching", "ready", "paused") or not (notice is None or isinstance(notice, str)) \
|
|
or not (sid is None or (isinstance(sid, int) and not isinstance(sid, bool))) or not isinstance(game, bool):
|
|
self.error("bad_message", "llm needs state switching|ready|paused, an optional string notice, integer id and boolean game")
|
|
return
|
|
w = self.worker
|
|
w.llm_state = state
|
|
if state == "ready":
|
|
w.llm_game = game
|
|
if w.scheduler:
|
|
w.scheduler.wake()
|
|
debug_log(f"llm {state}", "llm")
|
|
if state == "ready":
|
|
w.llm_notice, w.llm_idle_to = None, None
|
|
w.llm_ready.set()
|
|
return
|
|
w.llm_ready.clear()
|
|
if state == "switching":
|
|
w.llm_notice, w.llm_noticed = notice or None, False
|
|
w.llm_idle_to, w.llm_switch_id = self, sid
|
|
self._llm_check_idle()
|
|
else:
|
|
w.llm_ready.set() # paused: waiting requests stop waiting and answer nothing
|
|
|
|
def _llm_check_idle(self) -> None:
|
|
w = self.worker
|
|
if w.llm_state == "switching" and w.llm_active == 0 and w.llm_idle_to is not None:
|
|
to, w.llm_idle_to = w.llm_idle_to, None
|
|
to.send({"type": "llm_idle"} if w.llm_switch_id is None else {"type": "llm_idle", "id": w.llm_switch_id})
|
|
|
|
@contextlib.asynccontextmanager
|
|
async def _llm(self, notice_id: str | None = None):
|
|
"""Brain work that uses the LLM: it waits while the LLM switches (the first spoken request
|
|
that waits hears the notice once, under its own id), fails while paused or after waiting
|
|
`LLM_WAIT_S`, and counts as running so a switch waits for it."""
|
|
w = self.worker
|
|
while w.llm_state != "ready":
|
|
if w.llm_state == "paused":
|
|
raise _LlmPaused()
|
|
if notice_id and w.llm_notice and not w.llm_noticed:
|
|
w.llm_noticed = True
|
|
# The newline ends it as a sentence, so it is spoken now, not joined to the answer.
|
|
self._on_tts({"type": "tts", "id": notice_id, "text": w.llm_notice + "\n", "more": True})
|
|
try:
|
|
await asyncio.wait_for(w.llm_ready.wait(), LLM_WAIT_S)
|
|
except asyncio.TimeoutError:
|
|
# The LLM is still away (or a measuring server stands in): fail, never call it.
|
|
raise _LlmPaused() from None
|
|
w.llm_active += 1
|
|
try:
|
|
yield
|
|
finally:
|
|
w.llm_active -= 1
|
|
self._llm_check_idle()
|
|
|
|
def _on_prepare(self, msg: dict) -> None:
|
|
"""A call still being said: the brain gets ready for it. One prepare per user runs at a time;
|
|
while it runs only the newest text waits, and one about the same as the last one prepared
|
|
(or being prepared) is dropped. Nothing is sent back."""
|
|
from .brain import similar
|
|
got = self._strings(msg, "user", "text")
|
|
if got is None:
|
|
return
|
|
user, text = got
|
|
if self.worker.brain is None or not hasattr(self.worker.brain, "prepare"):
|
|
return
|
|
if user in self.last_prepared and similar(self.last_prepared[user], text):
|
|
return
|
|
if user in self.preparing:
|
|
self.preparing[user] = text
|
|
return
|
|
self.preparing[user] = None
|
|
asyncio.get_running_loop().create_task(self._prepare(user, text))
|
|
|
|
async def _prepare(self, user: str, text: str | None) -> None:
|
|
from .brain import similar
|
|
try:
|
|
while text is not None and not self.closed:
|
|
self.last_prepared[user] = text
|
|
try:
|
|
async with self._llm():
|
|
await asyncio.to_thread(self.worker.brain.prepare, user, text, bool(self.chat_channels))
|
|
except Exception as e: # the brain must never take the session down
|
|
debug_log(f"brain prepare failed: {type(e).__name__}", "llm")
|
|
newer, self.preparing[user] = self.preparing[user], None
|
|
text = newer if newer is not None and not similar(newer, self.last_prepared.get(user, "")) else None
|
|
finally:
|
|
self.preparing.pop(user, None)
|
|
|
|
def _on_addressed(self, msg: dict) -> None:
|
|
got = self._strings(msg, "id", "user", "text")
|
|
if got is None:
|
|
return
|
|
qid, user, text = got
|
|
if self.worker.brain is None:
|
|
self.error("brain_unavailable", "the worker was started without --brain", id=qid)
|
|
return
|
|
# Who is in the voice channel (people, and of them who may call), when the app says.
|
|
count = lambda k: msg[k] if isinstance(msg.get(k), int) and not isinstance(msg[k], bool) and msg[k] > 0 else None
|
|
people, callers = count("people"), count("callers")
|
|
|
|
async def judge():
|
|
try:
|
|
async with self._llm():
|
|
yes = await asyncio.to_thread(self.worker.brain.addressed, user, text, list(self.settings.hotwords),
|
|
people, callers)
|
|
except Exception as e: # the brain must never take the session down
|
|
debug_log(f"brain addressed {qid} failed: {type(e).__name__}", "llm")
|
|
yes = None
|
|
debug_log(f"brain addressed {qid} -> {yes}", "llm")
|
|
self.send({"type": "addressed_result", "id": qid, "addressed": bool(yes), "failed": yes is None})
|
|
asyncio.get_running_loop().create_task(judge())
|
|
|
|
def _on_rank(self, msg: dict) -> None:
|
|
qid, cands = msg.get("id"), msg.get("candidates")
|
|
ok = isinstance(qid, str) and isinstance(cands, list) and all(
|
|
isinstance(c, dict) and all(isinstance(c.get(k), str) for k in ("id", "user", "text")) for c in cands)
|
|
if not ok:
|
|
self.error("bad_message", "rank needs a string id and candidates of string id, user and text")
|
|
return
|
|
if self.worker.brain is None:
|
|
self.error("brain_unavailable", "the worker was started without --brain", id=qid)
|
|
return
|
|
cands = [{"id": c["id"], "user": c["user"], "text": c["text"]} for c in cands]
|
|
|
|
async def rank():
|
|
try:
|
|
async with self._llm():
|
|
order = await asyncio.to_thread(self.worker.brain.rank, cands)
|
|
except Exception as e:
|
|
debug_log(f"brain rank {qid} failed: {type(e).__name__}", "llm")
|
|
order = None
|
|
known = {c["id"] for c in cands}
|
|
order = [i for i in (order or []) if i in known] if order is not None else None
|
|
debug_log(f"brain rank {qid} candidates={len(cands)} ok={order is not None}", "llm")
|
|
self.send({"type": "rank_result", "id": qid, "order": order or [], "failed": order is None})
|
|
asyncio.get_running_loop().create_task(rank())
|
|
|
|
async def _reply(self, rid: str, user: str, text: str, via: str, speak: bool, early: bool = False,
|
|
channel: str | None = None, notes: list | None = None) -> None:
|
|
trace = _LiveTrace(self.worker.history, via, self.voice_channel if via == "voice" else channel, user, text)
|
|
try:
|
|
await self._reply_inner(rid, user, text, via, speak, early, channel, notes or [], trace=trace)
|
|
finally:
|
|
trace.finish("") # only when it did not get to its answer
|
|
self.thinking_replies.discard(rid)
|
|
self.cancelled_replies.discard(rid)
|
|
if not self._handed_over.pop(rid, False): # else _delivered counts it
|
|
self._reached(user)
|
|
|
|
def _reached(self, user: str) -> None:
|
|
waiting = self.worker.replies_unanswered
|
|
waiting[user] -= 1
|
|
if not waiting[user]:
|
|
del waiting[user]
|
|
|
|
def _delivered(self, rid: str, ok: bool) -> None:
|
|
"""The answer ``rid`` reached its asker (spoken, or posted by the app), or never will."""
|
|
got = self.deliveries.pop(rid, None)
|
|
if got is None:
|
|
return
|
|
user, turn, timer = got
|
|
if timer is not None:
|
|
timer.cancel()
|
|
if not ok:
|
|
# They never got this answer, so nothing it asked can be confirmed next.
|
|
w = self.worker
|
|
w.reply_turns[user] = max(w.reply_turns.get(user, 0), turn + 1)
|
|
debug_log(f"brain reply {rid} did not reach the asker", "llm")
|
|
self._reached(user)
|
|
|
|
def _on_reply_delivered(self, msg: dict) -> None:
|
|
rid, ok = msg.get("id"), msg.get("ok")
|
|
if not isinstance(rid, str) or not isinstance(ok, bool):
|
|
self.error("bad_message", "reply_delivered needs a string id and a boolean ok")
|
|
return
|
|
got = self.deliveries.get(rid)
|
|
if got is not None and got[2] is not None: # only chat answers wait for the app's report
|
|
self._delivered(rid, ok)
|
|
|
|
async def _reply_inner(self, rid: str, user: str, text: str, via: str, speak: bool, early: bool,
|
|
chat_channel: str | None = None, notes: list | None = None, *,
|
|
trace: _LiveTrace) -> None:
|
|
w = self.worker
|
|
channel = self.voice_channel
|
|
# Person first, then lane: a reply waiting for its asker's other answer holds no lane.
|
|
async with w.brain_people.setdefault(user, asyncio.Lock()), w.brain_lanes[via]:
|
|
if self.closed:
|
|
return
|
|
now = time.time()
|
|
if w.history is not None:
|
|
m = self.memory
|
|
history = await asyncio.to_thread(
|
|
w.history.context, user, text, now, m.scope(now), minutes=m.recent_minutes,
|
|
max_records=m.recent_records, limit=m.matches, budget_tokens=m.budget_tokens)
|
|
else:
|
|
from jarvis.memory.history import HistoryContext
|
|
history = HistoryContext()
|
|
turn = w.reply_turns[user] = w.reply_turns.get(user, 0) + (2 if early else 1)
|
|
if early:
|
|
debug_log(f"brain reply {rid} arrived before the previous answer: a turn is skipped",
|
|
"llm")
|
|
from .chat_bridge import ChatBridge
|
|
chat = ChatBridge(self, asyncio.get_running_loop(), turn, w.pending_deletes.setdefault(user, {}), user) \
|
|
if self.chat_channels else None
|
|
from .app_bridge import AppBridge
|
|
origin = (user, via, chat_channel if via == "chat" else None)
|
|
schedule = AppBridge(self, asyncio.get_running_loop(), "schedule", *origin)
|
|
notes_backend = AppBridge(self, asyncio.get_running_loop(), "note", *origin)
|
|
screen = AppBridge(self, asyncio.get_running_loop(), "screen", *origin)
|
|
trace.append(self._records_step(history, notes))
|
|
try:
|
|
async with self._llm(rid if speak else None):
|
|
w.answering += 1
|
|
try:
|
|
answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat,
|
|
schedule=schedule, notes=notes or [],
|
|
notes_backend=notes_backend, screen=screen,
|
|
trace=trace)
|
|
finally:
|
|
w.answering -= 1
|
|
if not w.answering and w.scheduler:
|
|
w.scheduler.wake() # held partials go on
|
|
except Exception as e: # the brain must never take the session down
|
|
debug_log(f"brain reply {rid} failed: {type(e).__name__}", "llm")
|
|
answer = None
|
|
answer = (answer or "").strip()
|
|
trace.finish(answer)
|
|
if not answer and rid in self.tts_requests:
|
|
self._on_tts({"type": "tts", "id": rid, "text": "", "more": False}) # ends the notice's speech
|
|
self.send({"type": "reply_text", "id": rid, "text": answer, "failed": not answer})
|
|
# The answer reaches them when spoken (tts_done) or when the app reports the post
|
|
# (reply_delivered); an unspoken voice answer has nothing more to wait for.
|
|
posted = via == "chat" and not speak
|
|
timer = asyncio.get_running_loop().call_later(w.delivery_timeout_s, self._delivered, rid, False) \
|
|
if posted else None
|
|
self.deliveries[rid] = (user, turn, timer)
|
|
self._handed_over[rid] = True
|
|
if self.closed or not answer or (speak and rid in self.cancelled_replies):
|
|
self._delivered(rid, False)
|
|
elif not speak and not posted:
|
|
self._delivered(rid, True)
|
|
if not answer:
|
|
return
|
|
if via == "voice" and channel and w.history is not None:
|
|
w.history.add_reply(channel, user, answer, time.time())
|
|
if speak and rid in self.cancelled_replies:
|
|
debug_log(f"brain reply {rid} not spoken: cancelled while answering", "llm")
|
|
self.send({"type": "tts_done", "id": rid, "chunks": 0, "cancelled": True, "failed": False})
|
|
elif speak:
|
|
self._on_tts({"type": "tts", "id": rid, "text": answer, "more": False})
|
|
|
|
def _tts_chunk_cb(self, rid: str):
|
|
def cb(seq: int, pcm: np.ndarray) -> None:
|
|
pcm = np.asarray(pcm, dtype="<i2")
|
|
rec = self.tts_recorded.get(rid)
|
|
if rec is not None:
|
|
rec["at"] = rec["at"] or time.time()
|
|
rec["pcm"].append(pcm.copy())
|
|
if self.tts_format["channels"] == 2:
|
|
pcm = np.repeat(pcm, 2)
|
|
self.send(encode_tts_frame(rid, seq, pcm.tobytes()))
|
|
return cb
|
|
|
|
def _tts_done(self, req: TtsRequest) -> None:
|
|
self.tts_requests.pop(req.id, None)
|
|
rec = self.tts_recorded.get(req.id)
|
|
if rec is not None:
|
|
rec["made"] = True
|
|
self._record_spoken(req.id)
|
|
self.send({"type": "tts_done", "id": req.id, "chunks": req.chunks, "cancelled": req.cancelled,
|
|
"failed": req.failed})
|
|
self._delivered(req.id, not (req.cancelled or req.failed))
|
|
|
|
def _on_tts_played(self, msg: dict) -> None:
|
|
"""How much of a spoken answer the app played: the recording keeps that much."""
|
|
rec = self.tts_recorded.get(msg.get("id"))
|
|
played = msg.get("played")
|
|
if rec is None or isinstance(played, bool) or not isinstance(played, int) or played < 0:
|
|
return
|
|
rec["played"], rec["cancelled"] = played, bool(msg.get("cancelled"))
|
|
last_ms = msg.get("last_ms") # how long the last sentence played before a stop
|
|
rec["last_ms"] = last_ms if isinstance(last_ms, int) and not isinstance(last_ms, bool) and last_ms >= 0 else None
|
|
self._record_spoken(msg["id"])
|
|
|
|
def _record_spoken(self, rid: str) -> None:
|
|
"""Keep a spoken answer once it is made and the app said how much of it was heard: the sentences
|
|
that played, the last one cut where a stop cut it."""
|
|
rec = self.tts_recorded.get(rid)
|
|
if rec is None or not rec["made"] or rec["played"] is None:
|
|
return
|
|
self.tts_recorded.pop(rid)
|
|
heard = list(rec["pcm"][:rec["played"]])
|
|
if heard and rec.get("last_ms") is not None:
|
|
heard[-1] = heard[-1][:rec["rate"] * rec["last_ms"] // 1000]
|
|
heard = [h for h in heard if len(h)]
|
|
if heard and self.worker.recorder is not None:
|
|
self.worker.recorder.spoke(rec["at"], rid, rec["text"], np.concatenate(heard), rec["rate"],
|
|
rec["cancelled"])
|
|
|
|
def _tts_error(self, req: TtsRequest, message: str) -> None:
|
|
self.error("tts_failed", message, id=req.id)
|