- Tool (chat_channels.spec.md): list, read, search, send, edit, react,
delete, pin, unpin. Offered only when the v2 brain gives the turn a chat
backend, and then on every turn. Channel by ID, name, part of a name or a
name inside the text; a message ID finds its channel; planner forms of the
action ("send_message") work. Read and search results are fenced as
untrusted data. delete only reports until called again with confirm=true
in a later turn of the same person.
- Worker: chat_channels from the app, ChatBridge reads/searches the history
store and sends chat_action, waiting for chat_action_result (15 s timeout,
failed on session close); per-person reply turns and pending deletes.
- App: channel switches edit/react (on) and delete/pin (off); chat_channels
sent on worker ready and after login; chat_action runs send, edit (own
messages only), react, delete, pin, unpin and answers with ids or the error.
- Engine: chatChannels left out of the catalogue without a backend; the
"router returned everything" check now tolerates that.
- live_discord: edit own, refuse editing the speaker's, react, pin (Discord
refuses in a voice channel's chat, reported as an error), delete, refuse an
unregistered channel; passed in 109.4 s.
- Eval (qwen2.5:7b, 3 runs): 10 of 12; never obeyed the "delete everything"
message and never deleted in the same turn.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
483 lines
17 KiB
Rust
483 lines
17 KiB
Rust
//! WebSocket client for the Python voice worker (`src/voice_worker/voice_worker.spec.md`).
|
|
//!
|
|
//! Keeps one connection open, says hello with the app's token, and reconnects with backoff when the
|
|
//! worker restarts. While disconnected, audio is dropped (the worker would not know the speaker's
|
|
//! state anyway) and every open TTS request ends as failed, so playback never waits forever.
|
|
|
|
use std::collections::HashSet;
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::Duration;
|
|
|
|
use futures_util::{SinkExt, StreamExt};
|
|
use serde::Deserialize;
|
|
use serde_json::{json, Map, Value};
|
|
use tokio::sync::{mpsc, watch, Notify};
|
|
use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
|
|
use tokio_tungstenite::tungstenite::Message;
|
|
|
|
use crate::frames::{decode_tts_frame, encode_audio_frame, KIND_TTS};
|
|
use crate::playback::TtsBackend;
|
|
use crate::{debug_log, error_log};
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub struct PcmFormat {
|
|
pub rate: u32,
|
|
pub channels: u8,
|
|
}
|
|
|
|
/// Discord voice after Opus decoding, and what the worker is asked to send back.
|
|
pub const DISCORD_PCM: PcmFormat = PcmFormat { rate: 48000, channels: 2 };
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct WorkerClientOptions {
|
|
pub url: String,
|
|
pub token: String,
|
|
pub audio: PcmFormat,
|
|
pub tts: PcmFormat,
|
|
pub stt: Map<String, Value>,
|
|
/// First reconnect delay; doubles up to `max_backoff`.
|
|
pub backoff: Duration,
|
|
pub max_backoff: Duration,
|
|
pub hello_timeout: Duration,
|
|
}
|
|
|
|
impl WorkerClientOptions {
|
|
pub fn new(url: impl Into<String>, token: impl Into<String>) -> Self {
|
|
Self {
|
|
url: url.into(),
|
|
token: token.into(),
|
|
audio: DISCORD_PCM,
|
|
tts: DISCORD_PCM,
|
|
stt: Map::new(),
|
|
backoff: Duration::from_millis(500),
|
|
max_backoff: Duration::from_secs(5),
|
|
hello_timeout: Duration::from_secs(5),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, PartialEq)]
|
|
pub struct HelloOk {
|
|
pub stt_ready: bool,
|
|
pub tts_ready: bool,
|
|
#[serde(default)]
|
|
pub stt: Map<String, Value>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
|
pub struct SpeechStart {
|
|
pub speaker: String,
|
|
pub utt: u32,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
|
pub struct Partial {
|
|
pub speaker: String,
|
|
pub utt: u32,
|
|
pub text: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
|
pub struct Final {
|
|
pub speaker: String,
|
|
pub utt: u32,
|
|
pub text: String,
|
|
pub end_reason: String,
|
|
pub audio_ms: u64,
|
|
pub wait_ms: u64,
|
|
pub stt_ms: u64,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
|
pub struct TtsDone {
|
|
pub id: String,
|
|
pub chunks: u32,
|
|
pub cancelled: bool,
|
|
pub failed: bool,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
|
pub struct WorkerError {
|
|
pub code: String,
|
|
pub message: String,
|
|
#[serde(default)]
|
|
pub id: Option<String>,
|
|
}
|
|
|
|
/// The brain's chat channel tool asks the app to act (voice_worker.spec.md, chat_channels.spec.md).
|
|
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
|
pub struct ChatAction {
|
|
pub id: String,
|
|
/// `send`, `edit`, `react`, `delete`, `pin` or `unpin`.
|
|
pub action: String,
|
|
pub channel: String,
|
|
#[serde(default)]
|
|
pub message_id: Option<String>,
|
|
#[serde(default)]
|
|
pub text: Option<String>,
|
|
#[serde(default)]
|
|
pub emoji: Option<String>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
pub enum WorkerEvent {
|
|
Ready(HelloOk),
|
|
Disconnected,
|
|
SpeechStart(SpeechStart),
|
|
Partial(Partial),
|
|
Final(Final),
|
|
TtsAudio { id: String, seq: u32, pcm: Vec<u8> },
|
|
TtsDone(TtsDone),
|
|
ChatAction(ChatAction),
|
|
Error(WorkerError),
|
|
}
|
|
|
|
struct State {
|
|
conn: Option<mpsc::UnboundedSender<Message>>,
|
|
stt: Map<String, Value>,
|
|
/// TTS requests sent and not yet done, so a lost connection can end them.
|
|
open_tts: HashSet<String>,
|
|
closed: bool,
|
|
dropped_frames: u64,
|
|
}
|
|
|
|
struct Inner {
|
|
opts: WorkerClientOptions,
|
|
state: Mutex<State>,
|
|
events: mpsc::UnboundedSender<WorkerEvent>,
|
|
ready: watch::Sender<Option<HelloOk>>,
|
|
wake: Notify,
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
pub struct WorkerClient {
|
|
inner: Arc<Inner>,
|
|
}
|
|
|
|
impl WorkerClient {
|
|
/// The client and the stream of worker events. Call `start()` to connect.
|
|
pub fn new(opts: WorkerClientOptions) -> (Self, mpsc::UnboundedReceiver<WorkerEvent>) {
|
|
let (events, rx) = mpsc::unbounded_channel();
|
|
let stt = opts.stt.clone();
|
|
let inner = Arc::new(Inner {
|
|
opts,
|
|
state: Mutex::new(State { conn: None, stt, open_tts: HashSet::new(), closed: false, dropped_frames: 0 }),
|
|
events,
|
|
ready: watch::channel(None).0,
|
|
wake: Notify::new(),
|
|
});
|
|
(Self { inner }, rx)
|
|
}
|
|
|
|
/// Start connecting; reconnects until `close()`.
|
|
pub fn start(&self) {
|
|
let inner = self.inner.clone();
|
|
tokio::spawn(async move { run(inner).await });
|
|
}
|
|
|
|
pub fn is_ready(&self) -> bool {
|
|
self.inner.ready.borrow().is_some()
|
|
}
|
|
|
|
/// Resolve once the worker has said hello_ok, or None after `timeout`.
|
|
pub async fn wait_ready(&self, timeout: Duration) -> Option<HelloOk> {
|
|
let mut rx = self.inner.ready.subscribe();
|
|
tokio::time::timeout(timeout, async {
|
|
loop {
|
|
if let Some(h) = rx.borrow_and_update().clone() {
|
|
return h;
|
|
}
|
|
if rx.changed().await.is_err() {
|
|
std::future::pending::<()>().await;
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.ok()
|
|
}
|
|
|
|
pub fn close(&self) {
|
|
{
|
|
let mut st = self.inner.state.lock().unwrap();
|
|
st.closed = true;
|
|
if let Some(tx) = st.conn.take() {
|
|
let _ = tx.send(Message::Close(None));
|
|
}
|
|
}
|
|
self.inner.wake.notify_waiters();
|
|
self.inner.wake.notify_one();
|
|
mark_disconnected(&self.inner);
|
|
}
|
|
|
|
fn send(&self, msg: Message) -> bool {
|
|
let st = self.inner.state.lock().unwrap();
|
|
match &st.conn {
|
|
Some(tx) if self.inner.ready.borrow().is_some() => tx.send(msg).is_ok(),
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
fn send_json(&self, v: Value) -> bool {
|
|
self.send(Message::text(v.to_string()))
|
|
}
|
|
|
|
/// One frame of a speaker's PCM in the hello "audio" format. Dropped while disconnected.
|
|
pub fn send_audio(&self, speaker: &str, pcm: &[i16]) {
|
|
let Some(frame) = encode_audio_frame(speaker, pcm) else { return };
|
|
if !self.send(Message::binary(frame)) {
|
|
self.inner.state.lock().unwrap().dropped_frames += 1;
|
|
}
|
|
}
|
|
|
|
pub fn speaking_end(&self, speaker: &str) {
|
|
self.send_json(json!({"type": "speaking_end", "speaker": speaker}));
|
|
}
|
|
|
|
pub fn speaker_leave(&self, speaker: &str) {
|
|
self.send_json(json!({"type": "speaker_leave", "speaker": speaker}));
|
|
}
|
|
|
|
// --- history (voice_worker.spec.md "History and brain"). Dropped while disconnected; the
|
|
// discord core sends the voice state again when the worker is ready. ----------------------
|
|
|
|
/// Jarvis is in (or moved to) this voice channel.
|
|
pub fn voice_join(&self, channel: u64) {
|
|
self.send_json(json!({"type": "voice_join", "channel": channel.to_string()}));
|
|
}
|
|
|
|
pub fn voice_leave(&self) {
|
|
self.send_json(json!({"type": "voice_leave"}));
|
|
}
|
|
|
|
pub fn member_join(&self, user: u64, name: &str) {
|
|
self.send_json(json!({"type": "member_join", "user": user.to_string(), "name": name}));
|
|
}
|
|
|
|
pub fn member_leave(&self, user: u64) {
|
|
self.send_json(json!({"type": "member_leave", "user": user.to_string()}));
|
|
}
|
|
|
|
/// A message of a registered chat channel. `to`: the user Jarvis answered, on its own messages.
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub fn chat_message(&self, channel: u64, message: u64, author: u64, name: &str, text: &str, ts: f64,
|
|
to: Option<u64>) {
|
|
let mut v = json!({"type": "chat_message", "channel": channel.to_string(), "message_id": message.to_string(),
|
|
"author": author.to_string(), "name": name, "text": text, "ts": ts});
|
|
if let Some(to) = to {
|
|
v["to"] = Value::String(to.to_string());
|
|
}
|
|
self.send_json(v);
|
|
}
|
|
|
|
pub fn chat_edit(&self, channel: u64, message: u64, text: &str) {
|
|
self.send_json(json!({"type": "chat_edit", "channel": channel.to_string(), "message_id": message.to_string(),
|
|
"text": text}));
|
|
}
|
|
|
|
/// The registered chat channels, their switches and the bot's own user ID, for the chat tool.
|
|
pub fn chat_channels(&self, bot: Option<u64>, channels: &[crate::settings::ChatChannelSettings]) {
|
|
let list: Vec<Value> = channels.iter().map(|c| json!({"id": c.id.to_string(), "name": c.name,
|
|
"description": c.description, "read": c.read, "send": c.send, "edit": c.edit, "react": c.react,
|
|
"delete": c.delete, "pin": c.pin})).collect();
|
|
let mut v = json!({"type": "chat_channels", "channels": list});
|
|
if let Some(b) = bot {
|
|
v["bot"] = Value::String(b.to_string());
|
|
}
|
|
self.send_json(v);
|
|
}
|
|
|
|
/// The outcome of a `chat_action`: the message ids it made or touched, or why it failed.
|
|
pub fn chat_action_result(&self, id: &str, result: Result<Vec<u64>, String>) {
|
|
let v = match result {
|
|
Ok(ids) => json!({"type": "chat_action_result", "id": id, "ok": true,
|
|
"message_ids": ids.iter().map(|i| i.to_string()).collect::<Vec<_>>()}),
|
|
Err(e) => json!({"type": "chat_action_result", "id": id, "ok": false, "error": e}),
|
|
};
|
|
self.send_json(v);
|
|
}
|
|
|
|
pub fn chat_delete(&self, channel: u64, message: u64) {
|
|
self.send_json(json!({"type": "chat_delete", "channel": channel.to_string(),
|
|
"message_id": message.to_string()}));
|
|
}
|
|
|
|
/// Change STT settings now and for every later connection.
|
|
pub fn configure(&self, changes: Map<String, Value>) {
|
|
self.inner.state.lock().unwrap().stt.extend(changes.clone());
|
|
let mut msg = changes;
|
|
msg.insert("type".into(), "config".into());
|
|
self.send_json(Value::Object(msg));
|
|
}
|
|
}
|
|
|
|
impl TtsBackend for WorkerClient {
|
|
/// Speak `text` as request `id`; with `more` the request stays open for more text. Returns false
|
|
/// (and the request ends as failed) when the worker is not connected.
|
|
fn tts(&self, id: &str, text: &str, more: bool) -> bool {
|
|
let fresh = self.inner.state.lock().unwrap().open_tts.insert(id.to_owned());
|
|
let sent = self.send_json(json!({"type": "tts", "id": id, "text": text, "more": more}));
|
|
if !sent {
|
|
let was_open = self.inner.state.lock().unwrap().open_tts.remove(id);
|
|
if fresh || was_open {
|
|
let _ = self.inner.events.send(WorkerEvent::TtsDone(TtsDone { id: id.into(), chunks: 0,
|
|
cancelled: false, failed: true }));
|
|
}
|
|
}
|
|
sent
|
|
}
|
|
|
|
fn tts_cancel(&self, id: &str) {
|
|
self.send_json(json!({"type": "tts_cancel", "id": id}));
|
|
}
|
|
}
|
|
|
|
fn mark_disconnected(inner: &Inner) {
|
|
let was = inner.ready.send_replace(None).is_some();
|
|
let (open, dropped) = {
|
|
let mut st = inner.state.lock().unwrap();
|
|
st.conn = None;
|
|
(std::mem::take(&mut st.open_tts), st.dropped_frames)
|
|
};
|
|
for id in open {
|
|
let _ = inner.events.send(WorkerEvent::TtsDone(TtsDone { id, chunks: 0, cancelled: false, failed: true }));
|
|
}
|
|
if was {
|
|
debug_log!("worker", "voice worker disconnected (dropped audio frames so far: {dropped})");
|
|
let _ = inner.events.send(WorkerEvent::Disconnected);
|
|
}
|
|
}
|
|
|
|
async fn run(inner: Arc<Inner>) {
|
|
let mut backoff = inner.opts.backoff;
|
|
loop {
|
|
if inner.state.lock().unwrap().closed {
|
|
return;
|
|
}
|
|
if connect_once(&inner).await {
|
|
backoff = inner.opts.backoff;
|
|
}
|
|
mark_disconnected(&inner);
|
|
if inner.state.lock().unwrap().closed {
|
|
return;
|
|
}
|
|
tokio::select! {
|
|
_ = tokio::time::sleep(backoff) => {}
|
|
_ = inner.wake.notified() => {}
|
|
}
|
|
backoff = (backoff * 2).min(inner.opts.max_backoff);
|
|
}
|
|
}
|
|
|
|
/// One connection's life. True if it got as far as hello_ok.
|
|
async fn connect_once(inner: &Arc<Inner>) -> bool {
|
|
let ws = match tokio_tungstenite::connect_async(inner.opts.url.as_str()).await {
|
|
Ok((ws, _)) => ws,
|
|
Err(e) => {
|
|
// Refused is normal while the worker is still loading its models.
|
|
debug_log!("worker", "voice worker connect failed: {e}");
|
|
return false;
|
|
}
|
|
};
|
|
let (mut write, mut read) = ws.split();
|
|
let stt = inner.state.lock().unwrap().stt.clone();
|
|
let o = &inner.opts;
|
|
let hello = json!({"type": "hello", "token": o.token,
|
|
"audio": {"rate": o.audio.rate, "channels": o.audio.channels},
|
|
"tts": {"rate": o.tts.rate, "channels": o.tts.channels}, "stt": stt});
|
|
if write.send(Message::text(hello.to_string())).await.is_err() {
|
|
return false;
|
|
}
|
|
let hello_ok = tokio::time::timeout(o.hello_timeout, async {
|
|
while let Some(Ok(msg)) = read.next().await {
|
|
match msg {
|
|
Message::Text(t) => {
|
|
let v: Value = serde_json::from_str(t.as_str()).ok()?;
|
|
if v["type"] == "hello_ok" {
|
|
return serde_json::from_value::<HelloOk>(v).ok();
|
|
}
|
|
}
|
|
Message::Close(Some(f)) => {
|
|
if matches!(f.code, CloseCode::Library(4400) | CloseCode::Library(4401)) {
|
|
error_log!("worker", "voice worker refused the connection ({} {})", u16::from(f.code), f.reason);
|
|
}
|
|
return None;
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
None
|
|
})
|
|
.await;
|
|
let Ok(Some(hello_ok)) = hello_ok else { return false };
|
|
|
|
let (tx, mut rx) = mpsc::unbounded_channel::<Message>();
|
|
{
|
|
let mut st = inner.state.lock().unwrap();
|
|
if st.closed {
|
|
return true;
|
|
}
|
|
st.conn = Some(tx);
|
|
}
|
|
debug_log!("worker", "voice worker ready stt={} tts={}", hello_ok.stt_ready, hello_ok.tts_ready);
|
|
inner.ready.send_replace(Some(hello_ok.clone()));
|
|
let _ = inner.events.send(WorkerEvent::Ready(hello_ok));
|
|
|
|
loop {
|
|
tokio::select! {
|
|
out = rx.recv() => match out {
|
|
Some(Message::Close(_)) | None => { let _ = write.close().await; break; }
|
|
Some(m) => if write.send(m).await.is_err() { break },
|
|
},
|
|
msg = read.next() => match msg {
|
|
Some(Ok(Message::Binary(b))) => on_binary(inner, &b),
|
|
Some(Ok(Message::Text(t))) => on_text(inner, t.as_str()),
|
|
Some(Ok(Message::Close(_))) | Some(Err(_)) | None => break,
|
|
Some(Ok(_)) => {}
|
|
},
|
|
}
|
|
}
|
|
true
|
|
}
|
|
|
|
fn on_binary(inner: &Inner, b: &[u8]) {
|
|
if b.first() != Some(&KIND_TTS) {
|
|
return;
|
|
}
|
|
if let Some(f) = decode_tts_frame(b) {
|
|
if inner.state.lock().unwrap().open_tts.contains(&f.id) {
|
|
let _ = inner.events.send(WorkerEvent::TtsAudio { id: f.id, seq: f.seq, pcm: f.pcm });
|
|
}
|
|
}
|
|
}
|
|
|
|
fn on_text(inner: &Inner, t: &str) {
|
|
let Ok(v) = serde_json::from_str::<Value>(t) else { return };
|
|
let kind = v["type"].as_str().unwrap_or_default().to_owned();
|
|
let ev = match kind.as_str() {
|
|
"speech_start" => serde_json::from_value(v).ok().map(WorkerEvent::SpeechStart),
|
|
"partial" => serde_json::from_value(v).ok().map(WorkerEvent::Partial),
|
|
"final" => serde_json::from_value(v).ok().map(WorkerEvent::Final),
|
|
"chat_action" => serde_json::from_value(v).ok().map(WorkerEvent::ChatAction),
|
|
"tts_done" => serde_json::from_value::<TtsDone>(v).ok().and_then(|d| {
|
|
inner.state.lock().unwrap().open_tts.remove(&d.id).then_some(WorkerEvent::TtsDone(d))
|
|
}),
|
|
"error" => serde_json::from_value::<WorkerError>(v).ok().map(|e| {
|
|
debug_log!("worker", "voice worker error {}", e.code);
|
|
// A request the worker refused outright never gets a tts_done.
|
|
if e.code == "tts_unavailable" {
|
|
if let Some(id) = &e.id {
|
|
if inner.state.lock().unwrap().open_tts.remove(id) {
|
|
let _ = inner.events.send(WorkerEvent::Error(e.clone()));
|
|
return WorkerEvent::TtsDone(TtsDone { id: id.clone(), chunks: 0, cancelled: false, failed: true });
|
|
}
|
|
}
|
|
}
|
|
WorkerEvent::Error(e)
|
|
}),
|
|
_ => None,
|
|
};
|
|
if let Some(ev) = ev {
|
|
let _ = inner.events.send(ev);
|
|
}
|
|
}
|