- the game stage reads the card again: other programs as at the last normal placement made with no game running, and a reserve of at least what grew since (the game and programs opened after that reading); with a game already running at the start (game detection now looks once before the LLM starts) it counts what the card holds now and leaves only the room for other programs, so it no longer pauses for counting the game twice - the retry count starts over whenever another stage is wanted, also when the game ends with the normal stage already running, so the next game tries again - the wait before the next try counts from when the failed switch and its way back are over - the worker client keeps, sends and readies a connection under one lock (llm, then state), so an LLM state set while a worker connects is not lost - a game watch dropped by a start that fails part way stops its thread Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
626 lines
24 KiB
Rust
626 lines
24 KiB
Rust
//! WebSocket client for the Python voice worker (`src/voice_worker/voice_worker.spec.md`).
|
|
//!
|
|
//! Keeps one connection open, says hello with the app's token, and reconnects with backoff when the
|
|
//! worker restarts. While disconnected, audio is dropped (the worker would not know the speaker's
|
|
//! state anyway) and every open TTS request ends as failed, so playback never waits forever.
|
|
|
|
use std::collections::HashSet;
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::Duration;
|
|
|
|
use futures_util::{SinkExt, StreamExt};
|
|
use serde::Deserialize;
|
|
use serde_json::{json, Map, Value};
|
|
use tokio::sync::{mpsc, watch, Notify};
|
|
use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
|
|
use tokio_tungstenite::tungstenite::Message;
|
|
|
|
use crate::frames::{decode_tts_frame, encode_audio_frame, KIND_TTS};
|
|
use crate::playback::TtsBackend;
|
|
use crate::{debug_log, error_log};
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub struct PcmFormat {
|
|
pub rate: u32,
|
|
pub channels: u8,
|
|
}
|
|
|
|
/// Discord voice after Opus decoding, and what the worker is asked to send back.
|
|
pub const DISCORD_PCM: PcmFormat = PcmFormat { rate: 48000, channels: 2 };
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct WorkerClientOptions {
|
|
pub url: String,
|
|
pub token: String,
|
|
pub audio: PcmFormat,
|
|
pub tts: PcmFormat,
|
|
pub stt: Map<String, Value>,
|
|
/// The worker's memory settings, sent in `hello` (voice_worker.spec.md).
|
|
pub memory: Map<String, Value>,
|
|
/// First reconnect delay; doubles up to `max_backoff`.
|
|
pub backoff: Duration,
|
|
pub max_backoff: Duration,
|
|
pub hello_timeout: Duration,
|
|
}
|
|
|
|
impl WorkerClientOptions {
|
|
pub fn new(url: impl Into<String>, token: impl Into<String>) -> Self {
|
|
Self {
|
|
url: url.into(),
|
|
token: token.into(),
|
|
audio: DISCORD_PCM,
|
|
tts: DISCORD_PCM,
|
|
stt: Map::new(),
|
|
memory: Map::new(),
|
|
backoff: Duration::from_millis(500),
|
|
max_backoff: Duration::from_secs(5),
|
|
hello_timeout: Duration::from_secs(5),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, PartialEq)]
|
|
pub struct HelloOk {
|
|
pub stt_ready: bool,
|
|
pub tts_ready: bool,
|
|
#[serde(default)]
|
|
pub stt: Map<String, Value>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
|
pub struct SpeechStart {
|
|
pub speaker: String,
|
|
pub utt: u32,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
|
pub struct Partial {
|
|
pub speaker: String,
|
|
pub utt: u32,
|
|
pub text: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
|
pub struct Final {
|
|
pub speaker: String,
|
|
pub utt: u32,
|
|
pub text: String,
|
|
pub end_reason: String,
|
|
pub audio_ms: u64,
|
|
pub wait_ms: u64,
|
|
pub stt_ms: u64,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
|
pub struct TtsDone {
|
|
pub id: String,
|
|
pub chunks: u32,
|
|
pub cancelled: bool,
|
|
pub failed: bool,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
|
pub struct WorkerError {
|
|
pub code: String,
|
|
pub message: String,
|
|
#[serde(default)]
|
|
pub id: Option<String>,
|
|
}
|
|
|
|
/// The brain's chat channel tool asks the app to act (voice_worker.spec.md, chat_channels.spec.md).
|
|
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
|
pub struct ChatAction {
|
|
pub id: String,
|
|
/// `send`, `edit`, `react`, `delete`, `pin` or `unpin`.
|
|
pub action: String,
|
|
pub channel: String,
|
|
#[serde(default)]
|
|
pub message_id: Option<String>,
|
|
#[serde(default)]
|
|
pub text: Option<String>,
|
|
#[serde(default)]
|
|
pub emoji: Option<String>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
pub enum WorkerEvent {
|
|
Ready(HelloOk),
|
|
Disconnected,
|
|
SpeechStart(SpeechStart),
|
|
Partial(Partial),
|
|
Final(Final),
|
|
TtsAudio { id: String, seq: u32, pcm: Vec<u8> },
|
|
TtsDone(TtsDone),
|
|
ChatAction(ChatAction),
|
|
/// The brain's answer to `reply` (empty and `failed` when it gave none).
|
|
ReplyText { id: String, text: String, failed: bool },
|
|
/// Was the utterance said to Jarvis? (a failed judgement is a no)
|
|
Addressed { id: String, addressed: bool },
|
|
/// Candidate ids, most important first (empty when the ranking failed).
|
|
Ranked { id: String, order: Vec<String> },
|
|
Error(WorkerError),
|
|
}
|
|
|
|
struct State {
|
|
conn: Option<mpsc::UnboundedSender<Message>>,
|
|
stt: Map<String, Value>,
|
|
/// TTS requests sent and not yet done, so a lost connection can end them.
|
|
open_tts: HashSet<String>,
|
|
closed: bool,
|
|
dropped_frames: u64,
|
|
}
|
|
|
|
struct Inner {
|
|
opts: WorkerClientOptions,
|
|
state: Mutex<State>,
|
|
events: mpsc::UnboundedSender<WorkerEvent>,
|
|
ready: watch::Sender<Option<HelloOk>>,
|
|
wake: Notify,
|
|
/// The last switch the brain reported idle for (`llm_idle` names it).
|
|
llm_idle: watch::Sender<u64>,
|
|
/// The LLM state the brain should have: sent again on every connection, so a worker that
|
|
/// missed it (a dropped connection, a restarted worker) catches up.
|
|
llm: Mutex<Option<Value>>,
|
|
/// The number of the latest switch.
|
|
llm_switches: std::sync::atomic::AtomicU64,
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
pub struct WorkerClient {
|
|
inner: Arc<Inner>,
|
|
}
|
|
|
|
impl WorkerClient {
|
|
/// The client and the stream of worker events. Call `start()` to connect.
|
|
pub fn new(opts: WorkerClientOptions) -> (Self, mpsc::UnboundedReceiver<WorkerEvent>) {
|
|
let (events, rx) = mpsc::unbounded_channel();
|
|
let stt = opts.stt.clone();
|
|
let inner = Arc::new(Inner {
|
|
opts,
|
|
state: Mutex::new(State { conn: None, stt, open_tts: HashSet::new(), closed: false, dropped_frames: 0 }),
|
|
events,
|
|
ready: watch::channel(None).0,
|
|
wake: Notify::new(),
|
|
llm_idle: watch::channel(0).0,
|
|
llm: Mutex::new(None),
|
|
llm_switches: std::sync::atomic::AtomicU64::new(0),
|
|
});
|
|
(Self { inner }, rx)
|
|
}
|
|
|
|
/// Start connecting; reconnects until `close()`.
|
|
pub fn start(&self) {
|
|
let inner = self.inner.clone();
|
|
tokio::spawn(async move { run(inner).await });
|
|
}
|
|
|
|
/// The worker's `hello_ok` while connected (STT and TTS readiness), else None.
|
|
pub fn hello(&self) -> Option<HelloOk> {
|
|
self.inner.ready.borrow().clone()
|
|
}
|
|
|
|
pub fn is_ready(&self) -> bool {
|
|
self.inner.ready.borrow().is_some()
|
|
}
|
|
|
|
/// Resolve once the worker has said hello_ok, or None after `timeout`.
|
|
pub async fn wait_ready(&self, timeout: Duration) -> Option<HelloOk> {
|
|
let mut rx = self.inner.ready.subscribe();
|
|
tokio::time::timeout(timeout, async {
|
|
loop {
|
|
if let Some(h) = rx.borrow_and_update().clone() {
|
|
return h;
|
|
}
|
|
if rx.changed().await.is_err() {
|
|
std::future::pending::<()>().await;
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.ok()
|
|
}
|
|
|
|
pub fn close(&self) {
|
|
{
|
|
let mut st = self.inner.state.lock().unwrap();
|
|
st.closed = true;
|
|
if let Some(tx) = st.conn.take() {
|
|
let _ = tx.send(Message::Close(None));
|
|
}
|
|
}
|
|
self.inner.wake.notify_waiters();
|
|
self.inner.wake.notify_one();
|
|
mark_disconnected(&self.inner);
|
|
}
|
|
|
|
fn send(&self, msg: Message) -> bool {
|
|
let st = self.inner.state.lock().unwrap();
|
|
match &st.conn {
|
|
Some(tx) if self.inner.ready.borrow().is_some() => tx.send(msg).is_ok(),
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
fn send_json(&self, v: Value) -> bool {
|
|
self.send(Message::text(v.to_string()))
|
|
}
|
|
|
|
/// One frame of a speaker's PCM in the hello "audio" format. Dropped while disconnected.
|
|
pub fn send_audio(&self, speaker: &str, pcm: &[i16]) {
|
|
let Some(frame) = encode_audio_frame(speaker, pcm) else { return };
|
|
if !self.send(Message::binary(frame)) {
|
|
self.inner.state.lock().unwrap().dropped_frames += 1;
|
|
}
|
|
}
|
|
|
|
pub fn speaking_end(&self, speaker: &str) {
|
|
self.send_json(json!({"type": "speaking_end", "speaker": speaker}));
|
|
}
|
|
|
|
pub fn speaker_leave(&self, speaker: &str) {
|
|
self.send_json(json!({"type": "speaker_leave", "speaker": speaker}));
|
|
}
|
|
|
|
// --- history (voice_worker.spec.md "History and brain"). Dropped while disconnected; the
|
|
// discord core sends the voice state again when the worker is ready. ----------------------
|
|
|
|
/// Jarvis is in (or moved to) this voice channel.
|
|
pub fn voice_join(&self, channel: u64) {
|
|
self.send_json(json!({"type": "voice_join", "channel": channel.to_string()}));
|
|
}
|
|
|
|
pub fn voice_leave(&self) {
|
|
self.send_json(json!({"type": "voice_leave"}));
|
|
}
|
|
|
|
pub fn member_join(&self, user: u64, name: &str) {
|
|
self.send_json(json!({"type": "member_join", "user": user.to_string(), "name": name}));
|
|
}
|
|
|
|
pub fn member_leave(&self, user: u64) {
|
|
self.send_json(json!({"type": "member_leave", "user": user.to_string()}));
|
|
}
|
|
|
|
/// A message of a registered chat channel. `to`: the user Jarvis answered, on its own messages.
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub fn chat_message(&self, channel: u64, message: u64, author: u64, name: &str, text: &str, ts: f64,
|
|
to: Option<u64>) {
|
|
let mut v = json!({"type": "chat_message", "channel": channel.to_string(), "message_id": message.to_string(),
|
|
"author": author.to_string(), "name": name, "text": text, "ts": ts});
|
|
if let Some(to) = to {
|
|
v["to"] = Value::String(to.to_string());
|
|
}
|
|
self.send_json(v);
|
|
}
|
|
|
|
pub fn chat_edit(&self, channel: u64, message: u64, text: &str) {
|
|
self.send_json(json!({"type": "chat_edit", "channel": channel.to_string(), "message_id": message.to_string(),
|
|
"text": text}));
|
|
}
|
|
|
|
/// The registered chat channels, their switches and the bot's own user ID, for the chat tool.
|
|
pub fn chat_channels(&self, bot: Option<u64>, channels: &[crate::settings::ChatChannelSettings]) {
|
|
let list: Vec<Value> = channels.iter().map(|c| json!({"id": c.id.to_string(), "name": c.name,
|
|
"description": c.description, "read": c.read, "send": c.send, "edit": c.edit, "react": c.react,
|
|
"delete": c.delete, "pin": c.pin})).collect();
|
|
let mut v = json!({"type": "chat_channels", "channels": list});
|
|
if let Some(b) = bot {
|
|
v["bot"] = Value::String(b.to_string());
|
|
}
|
|
self.send_json(v);
|
|
}
|
|
|
|
/// The outcome of a `chat_action`: the message ids it made or touched, or why it failed.
|
|
pub fn chat_action_result(&self, id: &str, result: Result<Vec<u64>, String>) {
|
|
let v = match result {
|
|
Ok(ids) => json!({"type": "chat_action_result", "id": id, "ok": true,
|
|
"message_ids": ids.iter().map(|i| i.to_string()).collect::<Vec<_>>()}),
|
|
Err(e) => json!({"type": "chat_action_result", "id": id, "ok": false, "error": e}),
|
|
};
|
|
self.send_json(v);
|
|
}
|
|
|
|
pub fn chat_delete(&self, channel: u64, message: u64) {
|
|
self.send_json(json!({"type": "chat_delete", "channel": channel.to_string(),
|
|
"message_id": message.to_string()}));
|
|
}
|
|
|
|
// --- brain (voice_worker.spec.md "History and brain", conversation.spec.md) -------------------
|
|
|
|
/// Ask the brain to answer `user`. With `speak` the answer is spoken under `id` (its audio and
|
|
/// `tts_done` come like a TTS request's); it ends as failed when it cannot be sent, the brain gives
|
|
/// no answer, or the worker refuses it.
|
|
/// `via`: `voice` or `chat` (the brain's history scope and where the answer is stored).
|
|
pub fn reply(&self, id: &str, user: u64, text: &str, via: &str, speak: bool) -> bool {
|
|
if speak {
|
|
self.inner.state.lock().unwrap().open_tts.insert(id.to_owned());
|
|
}
|
|
let sent = self.send_json(json!({"type": "reply", "id": id, "user": user.to_string(), "text": text,
|
|
"via": via, "speak": speak}));
|
|
if !sent && speak {
|
|
self.end_failed(id);
|
|
}
|
|
sent
|
|
}
|
|
|
|
/// Whether the chat answer `id` was posted (voice_worker.spec.md "History and brain"): a
|
|
/// question in an answer the person never saw cannot be confirmed in their next message.
|
|
pub fn reply_delivered(&self, id: &str, ok: bool) {
|
|
self.send_json(json!({"type": "reply_delivered", "id": id, "ok": ok}));
|
|
}
|
|
|
|
fn end_failed(&self, id: &str) {
|
|
if self.inner.state.lock().unwrap().open_tts.remove(id) {
|
|
let _ = self.inner.events.send(WorkerEvent::TtsDone(TtsDone { id: id.into(), chunks: 0, cancelled: false,
|
|
failed: true }));
|
|
}
|
|
}
|
|
|
|
/// Ask whether `text` was said to Jarvis (call mode `addressed`).
|
|
pub fn addressed(&self, id: &str, user: u64, text: &str) -> bool {
|
|
self.send_json(json!({"type": "addressed", "id": id, "user": user.to_string(), "text": text}))
|
|
}
|
|
|
|
/// Tell the brain the LLM switches (voice_worker.spec.md, "LLM switches"): it starts no new
|
|
/// LLM work, says `notice` once to the first spoken request that waits, and reports idle for
|
|
/// this switch, whose number is returned for `wait_llm_idle`.
|
|
pub fn llm_switching(&self, notice: Option<&str>) -> u64 {
|
|
let id = self.inner.llm_switches.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1;
|
|
let mut m = json!({"type": "llm", "state": "switching", "id": id});
|
|
if let Some(n) = notice {
|
|
m["notice"] = json!(n);
|
|
}
|
|
// Kept and sent under one lock, so a connection being set up sends this or a later state.
|
|
let mut llm = self.inner.llm.lock().unwrap();
|
|
*llm = Some(m.clone());
|
|
self.send_json(m);
|
|
id
|
|
}
|
|
|
|
/// Tell the brain the LLM is `ready` (answer what waited) or `paused` (answer nothing).
|
|
pub fn llm_state(&self, state: &str) -> bool {
|
|
let m = json!({"type": "llm", "state": state});
|
|
let mut llm = self.inner.llm.lock().unwrap();
|
|
*llm = Some(m.clone());
|
|
self.send_json(m)
|
|
}
|
|
|
|
/// Wait until the brain reports idle for switch `id` (also when it came before the wait); a
|
|
/// report for an earlier switch does not count. At most `t`.
|
|
pub async fn wait_llm_idle(&self, id: u64, t: Duration) -> bool {
|
|
let mut rx = self.inner.llm_idle.subscribe();
|
|
tokio::time::timeout(t, rx.wait_for(|got| *got >= id)).await.is_ok_and(|r| r.is_ok())
|
|
}
|
|
|
|
/// Tell the brain a call is being said (a partial transcript), so it gets ready to answer it.
|
|
pub fn prepare(&self, user: u64, text: &str) -> bool {
|
|
self.send_json(json!({"type": "prepare", "user": user.to_string(), "text": text}))
|
|
}
|
|
|
|
/// Ask the brain to rank tied candidates `(id, user, text)` by importance.
|
|
pub fn rank(&self, id: &str, candidates: &[(String, u64, String)]) -> bool {
|
|
let list: Vec<Value> = candidates.iter()
|
|
.map(|(cid, user, text)| json!({"id": cid, "user": user.to_string(), "text": text}))
|
|
.collect();
|
|
self.send_json(json!({"type": "rank", "id": id, "candidates": list}))
|
|
}
|
|
|
|
/// Change STT settings now and for every later connection.
|
|
pub fn configure(&self, changes: Map<String, Value>) {
|
|
self.inner.state.lock().unwrap().stt.extend(changes.clone());
|
|
let mut msg = changes;
|
|
msg.insert("type".into(), "config".into());
|
|
self.send_json(Value::Object(msg));
|
|
}
|
|
}
|
|
|
|
impl TtsBackend for WorkerClient {
|
|
/// Speak `text` as request `id`; with `more` the request stays open for more text. Returns false
|
|
/// (and the request ends as failed) when the worker is not connected.
|
|
fn tts(&self, id: &str, text: &str, more: bool) -> bool {
|
|
let fresh = self.inner.state.lock().unwrap().open_tts.insert(id.to_owned());
|
|
let sent = self.send_json(json!({"type": "tts", "id": id, "text": text, "more": more}));
|
|
if !sent {
|
|
let was_open = self.inner.state.lock().unwrap().open_tts.remove(id);
|
|
if fresh || was_open {
|
|
let _ = self.inner.events.send(WorkerEvent::TtsDone(TtsDone { id: id.into(), chunks: 0,
|
|
cancelled: false, failed: true }));
|
|
}
|
|
}
|
|
sent
|
|
}
|
|
|
|
fn tts_cancel(&self, id: &str) {
|
|
self.send_json(json!({"type": "tts_cancel", "id": id}));
|
|
}
|
|
}
|
|
|
|
fn mark_disconnected(inner: &Inner) {
|
|
let was = inner.ready.send_replace(None).is_some();
|
|
let (open, dropped) = {
|
|
let mut st = inner.state.lock().unwrap();
|
|
st.conn = None;
|
|
(std::mem::take(&mut st.open_tts), st.dropped_frames)
|
|
};
|
|
for id in open {
|
|
let _ = inner.events.send(WorkerEvent::TtsDone(TtsDone { id, chunks: 0, cancelled: false, failed: true }));
|
|
}
|
|
if was {
|
|
debug_log!("worker", "voice worker disconnected (dropped audio frames so far: {dropped})");
|
|
let _ = inner.events.send(WorkerEvent::Disconnected);
|
|
}
|
|
}
|
|
|
|
async fn run(inner: Arc<Inner>) {
|
|
let mut backoff = inner.opts.backoff;
|
|
loop {
|
|
if inner.state.lock().unwrap().closed {
|
|
return;
|
|
}
|
|
if connect_once(&inner).await {
|
|
backoff = inner.opts.backoff;
|
|
}
|
|
mark_disconnected(&inner);
|
|
if inner.state.lock().unwrap().closed {
|
|
return;
|
|
}
|
|
tokio::select! {
|
|
_ = tokio::time::sleep(backoff) => {}
|
|
_ = inner.wake.notified() => {}
|
|
}
|
|
backoff = (backoff * 2).min(inner.opts.max_backoff);
|
|
}
|
|
}
|
|
|
|
/// One connection's life. True if it got as far as hello_ok.
|
|
async fn connect_once(inner: &Arc<Inner>) -> bool {
|
|
let ws = match tokio_tungstenite::connect_async(inner.opts.url.as_str()).await {
|
|
Ok((ws, _)) => ws,
|
|
Err(e) => {
|
|
// Refused is normal while the worker is still loading its models.
|
|
debug_log!("worker", "voice worker connect failed: {e}");
|
|
return false;
|
|
}
|
|
};
|
|
let (mut write, mut read) = ws.split();
|
|
let stt = inner.state.lock().unwrap().stt.clone();
|
|
let o = &inner.opts;
|
|
let mut hello = json!({"type": "hello", "token": o.token,
|
|
"audio": {"rate": o.audio.rate, "channels": o.audio.channels},
|
|
"tts": {"rate": o.tts.rate, "channels": o.tts.channels}, "stt": stt});
|
|
if !o.memory.is_empty() {
|
|
hello["memory"] = Value::Object(o.memory.clone());
|
|
}
|
|
if write.send(Message::text(hello.to_string())).await.is_err() {
|
|
return false;
|
|
}
|
|
let hello_ok = tokio::time::timeout(o.hello_timeout, async {
|
|
while let Some(Ok(msg)) = read.next().await {
|
|
match msg {
|
|
Message::Text(t) => {
|
|
let v: Value = serde_json::from_str(t.as_str()).ok()?;
|
|
if v["type"] == "hello_ok" {
|
|
return serde_json::from_value::<HelloOk>(v).ok();
|
|
}
|
|
}
|
|
Message::Close(Some(f)) => {
|
|
if matches!(f.code, CloseCode::Library(4400) | CloseCode::Library(4401)) {
|
|
error_log!("worker", "voice worker refused the connection ({} {})", u16::from(f.code), f.reason);
|
|
}
|
|
return None;
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
None
|
|
})
|
|
.await;
|
|
let Ok(Some(hello_ok)) = hello_ok else { return false };
|
|
|
|
let (tx, mut rx) = mpsc::unbounded_channel::<Message>();
|
|
{
|
|
// Locks in the order llm, then state (as `llm_state` takes them): the latest LLM state goes
|
|
// first, and the connection is ready for sending before an LLM state set meanwhile is sent.
|
|
let llm = inner.llm.lock().unwrap();
|
|
let mut st = inner.state.lock().unwrap();
|
|
if st.closed {
|
|
return true;
|
|
}
|
|
if let Some(m) = llm.clone() {
|
|
let _ = tx.send(Message::text(m.to_string()));
|
|
}
|
|
st.conn = Some(tx);
|
|
inner.ready.send_replace(Some(hello_ok.clone()));
|
|
}
|
|
debug_log!("worker", "voice worker ready stt={} tts={}", hello_ok.stt_ready, hello_ok.tts_ready);
|
|
let _ = inner.events.send(WorkerEvent::Ready(hello_ok));
|
|
|
|
loop {
|
|
tokio::select! {
|
|
out = rx.recv() => match out {
|
|
Some(Message::Close(_)) | None => { let _ = write.close().await; break; }
|
|
Some(m) => if write.send(m).await.is_err() { break },
|
|
},
|
|
msg = read.next() => match msg {
|
|
Some(Ok(Message::Binary(b))) => on_binary(inner, &b),
|
|
Some(Ok(Message::Text(t))) => on_text(inner, t.as_str()),
|
|
Some(Ok(Message::Close(_))) | Some(Err(_)) | None => break,
|
|
Some(Ok(_)) => {}
|
|
},
|
|
}
|
|
}
|
|
true
|
|
}
|
|
|
|
fn on_binary(inner: &Inner, b: &[u8]) {
|
|
if b.first() != Some(&KIND_TTS) {
|
|
return;
|
|
}
|
|
if let Some(f) = decode_tts_frame(b) {
|
|
if inner.state.lock().unwrap().open_tts.contains(&f.id) {
|
|
let _ = inner.events.send(WorkerEvent::TtsAudio { id: f.id, seq: f.seq, pcm: f.pcm });
|
|
}
|
|
}
|
|
}
|
|
|
|
fn on_text(inner: &Inner, t: &str) {
|
|
let Ok(v) = serde_json::from_str::<Value>(t) else { return };
|
|
let kind = v["type"].as_str().unwrap_or_default().to_owned();
|
|
let ev = match kind.as_str() {
|
|
"speech_start" => serde_json::from_value(v).ok().map(WorkerEvent::SpeechStart),
|
|
"partial" => serde_json::from_value(v).ok().map(WorkerEvent::Partial),
|
|
"final" => serde_json::from_value(v).ok().map(WorkerEvent::Final),
|
|
"chat_action" => serde_json::from_value(v).ok().map(WorkerEvent::ChatAction),
|
|
"llm_idle" => {
|
|
if let Some(id) = v["id"].as_u64() {
|
|
inner.llm_idle.send_if_modified(|got| if id > *got { *got = id; true } else { false });
|
|
}
|
|
return;
|
|
}
|
|
"reply_text" => {
|
|
let id = v["id"].as_str().unwrap_or_default().to_owned();
|
|
let failed = v["failed"].as_bool().unwrap_or(true);
|
|
let text = v["text"].as_str().unwrap_or_default().to_owned();
|
|
// No answer means nothing will be spoken: end the speech the app is waiting for.
|
|
if failed && inner.state.lock().unwrap().open_tts.remove(&id) {
|
|
let _ = inner.events.send(WorkerEvent::ReplyText { id: id.clone(), text, failed });
|
|
let _ = inner.events.send(WorkerEvent::TtsDone(TtsDone { id, chunks: 0, cancelled: false, failed: true }));
|
|
return;
|
|
}
|
|
Some(WorkerEvent::ReplyText { id, text, failed })
|
|
}
|
|
"addressed_result" => Some(WorkerEvent::Addressed {
|
|
id: v["id"].as_str().unwrap_or_default().to_owned(),
|
|
addressed: v["addressed"].as_bool().unwrap_or(false) && !v["failed"].as_bool().unwrap_or(false),
|
|
}),
|
|
"rank_result" => Some(WorkerEvent::Ranked {
|
|
id: v["id"].as_str().unwrap_or_default().to_owned(),
|
|
order: if v["failed"].as_bool().unwrap_or(false) { vec![] } else {
|
|
v["order"].as_array().map(|a| a.iter().filter_map(|x| x.as_str().map(str::to_owned)).collect())
|
|
.unwrap_or_default()
|
|
},
|
|
}),
|
|
"tts_done" => serde_json::from_value::<TtsDone>(v).ok().and_then(|d| {
|
|
inner.state.lock().unwrap().open_tts.remove(&d.id).then_some(WorkerEvent::TtsDone(d))
|
|
}),
|
|
"error" => serde_json::from_value::<WorkerError>(v).ok().map(|e| {
|
|
// Always logged: the log screen and its error report must show what the worker refused.
|
|
error_log!("worker", "voice worker error {} id={}", e.code, e.id.as_deref().unwrap_or("-"));
|
|
// A request the worker refused outright never gets a tts_done.
|
|
if e.code == "tts_unavailable" || e.code == "brain_unavailable" {
|
|
if let Some(id) = &e.id {
|
|
if inner.state.lock().unwrap().open_tts.remove(id) {
|
|
let _ = inner.events.send(WorkerEvent::Error(e.clone()));
|
|
return WorkerEvent::TtsDone(TtsDone { id: id.clone(), chunks: 0, cancelled: false, failed: true });
|
|
}
|
|
}
|
|
}
|
|
WorkerEvent::Error(e)
|
|
}),
|
|
_ => None,
|
|
};
|
|
if let Some(ev) = ev {
|
|
let _ = inner.events.send(ev);
|
|
}
|
|
}
|