//! WebSocket client for the Python voice worker (`src/voice_worker/voice_worker.spec.md`). //! //! Keeps one connection open, says hello with the app's token, and reconnects with backoff when the //! worker restarts. While disconnected, audio is dropped (the worker would not know the speaker's //! state anyway) and every open TTS request ends as failed, so playback never waits forever. use std::collections::HashSet; use std::sync::{Arc, Mutex}; use std::time::Duration; use futures_util::{SinkExt, StreamExt}; use serde::Deserialize; use serde_json::{json, Map, Value}; use tokio::sync::{mpsc, watch, Notify}; use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode; use tokio_tungstenite::tungstenite::Message; use crate::frames::{decode_tts_frame, encode_audio_frame, KIND_TTS}; use crate::playback::TtsBackend; use crate::{debug_log, error_log}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct PcmFormat { pub rate: u32, pub channels: u8, } /// Discord voice after Opus decoding, and what the worker is asked to send back. pub const DISCORD_PCM: PcmFormat = PcmFormat { rate: 48000, channels: 2 }; #[derive(Debug, Clone)] pub struct WorkerClientOptions { pub url: String, pub token: String, pub audio: PcmFormat, pub tts: PcmFormat, pub stt: Map, /// First reconnect delay; doubles up to `max_backoff`. pub backoff: Duration, pub max_backoff: Duration, pub hello_timeout: Duration, } impl WorkerClientOptions { pub fn new(url: impl Into, token: impl Into) -> Self { Self { url: url.into(), token: token.into(), audio: DISCORD_PCM, tts: DISCORD_PCM, stt: Map::new(), backoff: Duration::from_millis(500), max_backoff: Duration::from_secs(5), hello_timeout: Duration::from_secs(5), } } } #[derive(Debug, Clone, Deserialize, PartialEq)] pub struct HelloOk { pub stt_ready: bool, pub tts_ready: bool, #[serde(default)] pub stt: Map, } #[derive(Debug, Clone, Deserialize, PartialEq, Eq)] pub struct SpeechStart { pub speaker: String, pub utt: u32, } #[derive(Debug, Clone, Deserialize, PartialEq, Eq)] pub struct Partial { pub speaker: String, pub utt: u32, pub text: String, } #[derive(Debug, Clone, Deserialize, PartialEq, Eq)] pub struct Final { pub speaker: String, pub utt: u32, pub text: String, pub end_reason: String, pub audio_ms: u64, pub wait_ms: u64, pub stt_ms: u64, } #[derive(Debug, Clone, Deserialize, PartialEq, Eq)] pub struct TtsDone { pub id: String, pub chunks: u32, pub cancelled: bool, pub failed: bool, } #[derive(Debug, Clone, Deserialize, PartialEq, Eq)] pub struct WorkerError { pub code: String, pub message: String, #[serde(default)] pub id: Option, } /// The brain's chat channel tool asks the app to act (voice_worker.spec.md, chat_channels.spec.md). #[derive(Debug, Clone, Deserialize, PartialEq, Eq)] pub struct ChatAction { pub id: String, /// `send`, `edit`, `react`, `delete`, `pin` or `unpin`. pub action: String, pub channel: String, #[serde(default)] pub message_id: Option, #[serde(default)] pub text: Option, #[serde(default)] pub emoji: Option, } #[derive(Debug, Clone, PartialEq)] pub enum WorkerEvent { Ready(HelloOk), Disconnected, SpeechStart(SpeechStart), Partial(Partial), Final(Final), TtsAudio { id: String, seq: u32, pcm: Vec }, TtsDone(TtsDone), ChatAction(ChatAction), Error(WorkerError), } struct State { conn: Option>, stt: Map, /// TTS requests sent and not yet done, so a lost connection can end them. open_tts: HashSet, closed: bool, dropped_frames: u64, } struct Inner { opts: WorkerClientOptions, state: Mutex, events: mpsc::UnboundedSender, ready: watch::Sender>, wake: Notify, } #[derive(Clone)] pub struct WorkerClient { inner: Arc, } impl WorkerClient { /// The client and the stream of worker events. Call `start()` to connect. pub fn new(opts: WorkerClientOptions) -> (Self, mpsc::UnboundedReceiver) { let (events, rx) = mpsc::unbounded_channel(); let stt = opts.stt.clone(); let inner = Arc::new(Inner { opts, state: Mutex::new(State { conn: None, stt, open_tts: HashSet::new(), closed: false, dropped_frames: 0 }), events, ready: watch::channel(None).0, wake: Notify::new(), }); (Self { inner }, rx) } /// Start connecting; reconnects until `close()`. pub fn start(&self) { let inner = self.inner.clone(); tokio::spawn(async move { run(inner).await }); } pub fn is_ready(&self) -> bool { self.inner.ready.borrow().is_some() } /// Resolve once the worker has said hello_ok, or None after `timeout`. pub async fn wait_ready(&self, timeout: Duration) -> Option { let mut rx = self.inner.ready.subscribe(); tokio::time::timeout(timeout, async { loop { if let Some(h) = rx.borrow_and_update().clone() { return h; } if rx.changed().await.is_err() { std::future::pending::<()>().await; } } }) .await .ok() } pub fn close(&self) { { let mut st = self.inner.state.lock().unwrap(); st.closed = true; if let Some(tx) = st.conn.take() { let _ = tx.send(Message::Close(None)); } } self.inner.wake.notify_waiters(); self.inner.wake.notify_one(); mark_disconnected(&self.inner); } fn send(&self, msg: Message) -> bool { let st = self.inner.state.lock().unwrap(); match &st.conn { Some(tx) if self.inner.ready.borrow().is_some() => tx.send(msg).is_ok(), _ => false, } } fn send_json(&self, v: Value) -> bool { self.send(Message::text(v.to_string())) } /// One frame of a speaker's PCM in the hello "audio" format. Dropped while disconnected. pub fn send_audio(&self, speaker: &str, pcm: &[i16]) { let Some(frame) = encode_audio_frame(speaker, pcm) else { return }; if !self.send(Message::binary(frame)) { self.inner.state.lock().unwrap().dropped_frames += 1; } } pub fn speaking_end(&self, speaker: &str) { self.send_json(json!({"type": "speaking_end", "speaker": speaker})); } pub fn speaker_leave(&self, speaker: &str) { self.send_json(json!({"type": "speaker_leave", "speaker": speaker})); } // --- history (voice_worker.spec.md "History and brain"). Dropped while disconnected; the // discord core sends the voice state again when the worker is ready. ---------------------- /// Jarvis is in (or moved to) this voice channel. pub fn voice_join(&self, channel: u64) { self.send_json(json!({"type": "voice_join", "channel": channel.to_string()})); } pub fn voice_leave(&self) { self.send_json(json!({"type": "voice_leave"})); } pub fn member_join(&self, user: u64, name: &str) { self.send_json(json!({"type": "member_join", "user": user.to_string(), "name": name})); } pub fn member_leave(&self, user: u64) { self.send_json(json!({"type": "member_leave", "user": user.to_string()})); } /// A message of a registered chat channel. `to`: the user Jarvis answered, on its own messages. #[allow(clippy::too_many_arguments)] pub fn chat_message(&self, channel: u64, message: u64, author: u64, name: &str, text: &str, ts: f64, to: Option) { let mut v = json!({"type": "chat_message", "channel": channel.to_string(), "message_id": message.to_string(), "author": author.to_string(), "name": name, "text": text, "ts": ts}); if let Some(to) = to { v["to"] = Value::String(to.to_string()); } self.send_json(v); } pub fn chat_edit(&self, channel: u64, message: u64, text: &str) { self.send_json(json!({"type": "chat_edit", "channel": channel.to_string(), "message_id": message.to_string(), "text": text})); } /// The registered chat channels, their switches and the bot's own user ID, for the chat tool. pub fn chat_channels(&self, bot: Option, channels: &[crate::settings::ChatChannelSettings]) { let list: Vec = channels.iter().map(|c| json!({"id": c.id.to_string(), "name": c.name, "description": c.description, "read": c.read, "send": c.send, "edit": c.edit, "react": c.react, "delete": c.delete, "pin": c.pin})).collect(); let mut v = json!({"type": "chat_channels", "channels": list}); if let Some(b) = bot { v["bot"] = Value::String(b.to_string()); } self.send_json(v); } /// The outcome of a `chat_action`: the message ids it made or touched, or why it failed. pub fn chat_action_result(&self, id: &str, result: Result, String>) { let v = match result { Ok(ids) => json!({"type": "chat_action_result", "id": id, "ok": true, "message_ids": ids.iter().map(|i| i.to_string()).collect::>()}), Err(e) => json!({"type": "chat_action_result", "id": id, "ok": false, "error": e}), }; self.send_json(v); } pub fn chat_delete(&self, channel: u64, message: u64) { self.send_json(json!({"type": "chat_delete", "channel": channel.to_string(), "message_id": message.to_string()})); } /// Change STT settings now and for every later connection. pub fn configure(&self, changes: Map) { self.inner.state.lock().unwrap().stt.extend(changes.clone()); let mut msg = changes; msg.insert("type".into(), "config".into()); self.send_json(Value::Object(msg)); } } impl TtsBackend for WorkerClient { /// Speak `text` as request `id`; with `more` the request stays open for more text. Returns false /// (and the request ends as failed) when the worker is not connected. fn tts(&self, id: &str, text: &str, more: bool) -> bool { let fresh = self.inner.state.lock().unwrap().open_tts.insert(id.to_owned()); let sent = self.send_json(json!({"type": "tts", "id": id, "text": text, "more": more})); if !sent { let was_open = self.inner.state.lock().unwrap().open_tts.remove(id); if fresh || was_open { let _ = self.inner.events.send(WorkerEvent::TtsDone(TtsDone { id: id.into(), chunks: 0, cancelled: false, failed: true })); } } sent } fn tts_cancel(&self, id: &str) { self.send_json(json!({"type": "tts_cancel", "id": id})); } } fn mark_disconnected(inner: &Inner) { let was = inner.ready.send_replace(None).is_some(); let (open, dropped) = { let mut st = inner.state.lock().unwrap(); st.conn = None; (std::mem::take(&mut st.open_tts), st.dropped_frames) }; for id in open { let _ = inner.events.send(WorkerEvent::TtsDone(TtsDone { id, chunks: 0, cancelled: false, failed: true })); } if was { debug_log!("worker", "voice worker disconnected (dropped audio frames so far: {dropped})"); let _ = inner.events.send(WorkerEvent::Disconnected); } } async fn run(inner: Arc) { let mut backoff = inner.opts.backoff; loop { if inner.state.lock().unwrap().closed { return; } if connect_once(&inner).await { backoff = inner.opts.backoff; } mark_disconnected(&inner); if inner.state.lock().unwrap().closed { return; } tokio::select! { _ = tokio::time::sleep(backoff) => {} _ = inner.wake.notified() => {} } backoff = (backoff * 2).min(inner.opts.max_backoff); } } /// One connection's life. True if it got as far as hello_ok. async fn connect_once(inner: &Arc) -> bool { let ws = match tokio_tungstenite::connect_async(inner.opts.url.as_str()).await { Ok((ws, _)) => ws, Err(e) => { // Refused is normal while the worker is still loading its models. debug_log!("worker", "voice worker connect failed: {e}"); return false; } }; let (mut write, mut read) = ws.split(); let stt = inner.state.lock().unwrap().stt.clone(); let o = &inner.opts; let hello = json!({"type": "hello", "token": o.token, "audio": {"rate": o.audio.rate, "channels": o.audio.channels}, "tts": {"rate": o.tts.rate, "channels": o.tts.channels}, "stt": stt}); if write.send(Message::text(hello.to_string())).await.is_err() { return false; } let hello_ok = tokio::time::timeout(o.hello_timeout, async { while let Some(Ok(msg)) = read.next().await { match msg { Message::Text(t) => { let v: Value = serde_json::from_str(t.as_str()).ok()?; if v["type"] == "hello_ok" { return serde_json::from_value::(v).ok(); } } Message::Close(Some(f)) => { if matches!(f.code, CloseCode::Library(4400) | CloseCode::Library(4401)) { error_log!("worker", "voice worker refused the connection ({} {})", u16::from(f.code), f.reason); } return None; } _ => {} } } None }) .await; let Ok(Some(hello_ok)) = hello_ok else { return false }; let (tx, mut rx) = mpsc::unbounded_channel::(); { let mut st = inner.state.lock().unwrap(); if st.closed { return true; } st.conn = Some(tx); } debug_log!("worker", "voice worker ready stt={} tts={}", hello_ok.stt_ready, hello_ok.tts_ready); inner.ready.send_replace(Some(hello_ok.clone())); let _ = inner.events.send(WorkerEvent::Ready(hello_ok)); loop { tokio::select! { out = rx.recv() => match out { Some(Message::Close(_)) | None => { let _ = write.close().await; break; } Some(m) => if write.send(m).await.is_err() { break }, }, msg = read.next() => match msg { Some(Ok(Message::Binary(b))) => on_binary(inner, &b), Some(Ok(Message::Text(t))) => on_text(inner, t.as_str()), Some(Ok(Message::Close(_))) | Some(Err(_)) | None => break, Some(Ok(_)) => {} }, } } true } fn on_binary(inner: &Inner, b: &[u8]) { if b.first() != Some(&KIND_TTS) { return; } if let Some(f) = decode_tts_frame(b) { if inner.state.lock().unwrap().open_tts.contains(&f.id) { let _ = inner.events.send(WorkerEvent::TtsAudio { id: f.id, seq: f.seq, pcm: f.pcm }); } } } fn on_text(inner: &Inner, t: &str) { let Ok(v) = serde_json::from_str::(t) else { return }; let kind = v["type"].as_str().unwrap_or_default().to_owned(); let ev = match kind.as_str() { "speech_start" => serde_json::from_value(v).ok().map(WorkerEvent::SpeechStart), "partial" => serde_json::from_value(v).ok().map(WorkerEvent::Partial), "final" => serde_json::from_value(v).ok().map(WorkerEvent::Final), "chat_action" => serde_json::from_value(v).ok().map(WorkerEvent::ChatAction), "tts_done" => serde_json::from_value::(v).ok().and_then(|d| { inner.state.lock().unwrap().open_tts.remove(&d.id).then_some(WorkerEvent::TtsDone(d)) }), "error" => serde_json::from_value::(v).ok().map(|e| { debug_log!("worker", "voice worker error {}", e.code); // A request the worker refused outright never gets a tts_done. if e.code == "tts_unavailable" { if let Some(id) = &e.id { if inner.state.lock().unwrap().open_tts.remove(id) { let _ = inner.events.send(WorkerEvent::Error(e.clone())); return WorkerEvent::TtsDone(TtsDone { id: id.clone(), chunks: 0, cancelled: false, failed: true }); } } } WorkerEvent::Error(e) }), _ => None, }; if let Some(ev) = ev { let _ = inner.events.send(ev); } }