Files
javis_bot/app/src/worker_client.rs
javis-bot fe4e296361 feat(chat): chatChannels tool for the registered Discord chat channels
- Tool (chat_channels.spec.md): list, read, search, send, edit, react,
  delete, pin, unpin. Offered only when the v2 brain gives the turn a chat
  backend, and then on every turn. Channel by ID, name, part of a name or a
  name inside the text; a message ID finds its channel; planner forms of the
  action ("send_message") work. Read and search results are fenced as
  untrusted data. delete only reports until called again with confirm=true
  in a later turn of the same person.
- Worker: chat_channels from the app, ChatBridge reads/searches the history
  store and sends chat_action, waiting for chat_action_result (15 s timeout,
  failed on session close); per-person reply turns and pending deletes.
- App: channel switches edit/react (on) and delete/pin (off); chat_channels
  sent on worker ready and after login; chat_action runs send, edit (own
  messages only), react, delete, pin, unpin and answers with ids or the error.
- Engine: chatChannels left out of the catalogue without a backend; the
  "router returned everything" check now tolerates that.
- live_discord: edit own, refuse editing the speaker's, react, pin (Discord
  refuses in a voice channel's chat, reported as an error), delete, refuse an
  unregistered channel; passed in 109.4 s.
- Eval (qwen2.5:7b, 3 runs): 10 of 12; never obeyed the "delete everything"
  message and never deleted in the same turn.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 14:01:33 +09:00

483 lines
17 KiB
Rust

//! WebSocket client for the Python voice worker (`src/voice_worker/voice_worker.spec.md`).
//!
//! Keeps one connection open, says hello with the app's token, and reconnects with backoff when the
//! worker restarts. While disconnected, audio is dropped (the worker would not know the speaker's
//! state anyway) and every open TTS request ends as failed, so playback never waits forever.
use std::collections::HashSet;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use futures_util::{SinkExt, StreamExt};
use serde::Deserialize;
use serde_json::{json, Map, Value};
use tokio::sync::{mpsc, watch, Notify};
use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
use tokio_tungstenite::tungstenite::Message;
use crate::frames::{decode_tts_frame, encode_audio_frame, KIND_TTS};
use crate::playback::TtsBackend;
use crate::{debug_log, error_log};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PcmFormat {
pub rate: u32,
pub channels: u8,
}
/// Discord voice after Opus decoding, and what the worker is asked to send back.
pub const DISCORD_PCM: PcmFormat = PcmFormat { rate: 48000, channels: 2 };
#[derive(Debug, Clone)]
pub struct WorkerClientOptions {
pub url: String,
pub token: String,
pub audio: PcmFormat,
pub tts: PcmFormat,
pub stt: Map<String, Value>,
/// First reconnect delay; doubles up to `max_backoff`.
pub backoff: Duration,
pub max_backoff: Duration,
pub hello_timeout: Duration,
}
impl WorkerClientOptions {
pub fn new(url: impl Into<String>, token: impl Into<String>) -> Self {
Self {
url: url.into(),
token: token.into(),
audio: DISCORD_PCM,
tts: DISCORD_PCM,
stt: Map::new(),
backoff: Duration::from_millis(500),
max_backoff: Duration::from_secs(5),
hello_timeout: Duration::from_secs(5),
}
}
}
#[derive(Debug, Clone, Deserialize, PartialEq)]
pub struct HelloOk {
pub stt_ready: bool,
pub tts_ready: bool,
#[serde(default)]
pub stt: Map<String, Value>,
}
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
pub struct SpeechStart {
pub speaker: String,
pub utt: u32,
}
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
pub struct Partial {
pub speaker: String,
pub utt: u32,
pub text: String,
}
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
pub struct Final {
pub speaker: String,
pub utt: u32,
pub text: String,
pub end_reason: String,
pub audio_ms: u64,
pub wait_ms: u64,
pub stt_ms: u64,
}
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
pub struct TtsDone {
pub id: String,
pub chunks: u32,
pub cancelled: bool,
pub failed: bool,
}
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
pub struct WorkerError {
pub code: String,
pub message: String,
#[serde(default)]
pub id: Option<String>,
}
/// The brain's chat channel tool asks the app to act (voice_worker.spec.md, chat_channels.spec.md).
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
pub struct ChatAction {
pub id: String,
/// `send`, `edit`, `react`, `delete`, `pin` or `unpin`.
pub action: String,
pub channel: String,
#[serde(default)]
pub message_id: Option<String>,
#[serde(default)]
pub text: Option<String>,
#[serde(default)]
pub emoji: Option<String>,
}
#[derive(Debug, Clone, PartialEq)]
pub enum WorkerEvent {
Ready(HelloOk),
Disconnected,
SpeechStart(SpeechStart),
Partial(Partial),
Final(Final),
TtsAudio { id: String, seq: u32, pcm: Vec<u8> },
TtsDone(TtsDone),
ChatAction(ChatAction),
Error(WorkerError),
}
struct State {
conn: Option<mpsc::UnboundedSender<Message>>,
stt: Map<String, Value>,
/// TTS requests sent and not yet done, so a lost connection can end them.
open_tts: HashSet<String>,
closed: bool,
dropped_frames: u64,
}
struct Inner {
opts: WorkerClientOptions,
state: Mutex<State>,
events: mpsc::UnboundedSender<WorkerEvent>,
ready: watch::Sender<Option<HelloOk>>,
wake: Notify,
}
#[derive(Clone)]
pub struct WorkerClient {
inner: Arc<Inner>,
}
impl WorkerClient {
/// The client and the stream of worker events. Call `start()` to connect.
pub fn new(opts: WorkerClientOptions) -> (Self, mpsc::UnboundedReceiver<WorkerEvent>) {
let (events, rx) = mpsc::unbounded_channel();
let stt = opts.stt.clone();
let inner = Arc::new(Inner {
opts,
state: Mutex::new(State { conn: None, stt, open_tts: HashSet::new(), closed: false, dropped_frames: 0 }),
events,
ready: watch::channel(None).0,
wake: Notify::new(),
});
(Self { inner }, rx)
}
/// Start connecting; reconnects until `close()`.
pub fn start(&self) {
let inner = self.inner.clone();
tokio::spawn(async move { run(inner).await });
}
pub fn is_ready(&self) -> bool {
self.inner.ready.borrow().is_some()
}
/// Resolve once the worker has said hello_ok, or None after `timeout`.
pub async fn wait_ready(&self, timeout: Duration) -> Option<HelloOk> {
let mut rx = self.inner.ready.subscribe();
tokio::time::timeout(timeout, async {
loop {
if let Some(h) = rx.borrow_and_update().clone() {
return h;
}
if rx.changed().await.is_err() {
std::future::pending::<()>().await;
}
}
})
.await
.ok()
}
pub fn close(&self) {
{
let mut st = self.inner.state.lock().unwrap();
st.closed = true;
if let Some(tx) = st.conn.take() {
let _ = tx.send(Message::Close(None));
}
}
self.inner.wake.notify_waiters();
self.inner.wake.notify_one();
mark_disconnected(&self.inner);
}
fn send(&self, msg: Message) -> bool {
let st = self.inner.state.lock().unwrap();
match &st.conn {
Some(tx) if self.inner.ready.borrow().is_some() => tx.send(msg).is_ok(),
_ => false,
}
}
fn send_json(&self, v: Value) -> bool {
self.send(Message::text(v.to_string()))
}
/// One frame of a speaker's PCM in the hello "audio" format. Dropped while disconnected.
pub fn send_audio(&self, speaker: &str, pcm: &[i16]) {
let Some(frame) = encode_audio_frame(speaker, pcm) else { return };
if !self.send(Message::binary(frame)) {
self.inner.state.lock().unwrap().dropped_frames += 1;
}
}
pub fn speaking_end(&self, speaker: &str) {
self.send_json(json!({"type": "speaking_end", "speaker": speaker}));
}
pub fn speaker_leave(&self, speaker: &str) {
self.send_json(json!({"type": "speaker_leave", "speaker": speaker}));
}
// --- history (voice_worker.spec.md "History and brain"). Dropped while disconnected; the
// discord core sends the voice state again when the worker is ready. ----------------------
/// Jarvis is in (or moved to) this voice channel.
pub fn voice_join(&self, channel: u64) {
self.send_json(json!({"type": "voice_join", "channel": channel.to_string()}));
}
pub fn voice_leave(&self) {
self.send_json(json!({"type": "voice_leave"}));
}
pub fn member_join(&self, user: u64, name: &str) {
self.send_json(json!({"type": "member_join", "user": user.to_string(), "name": name}));
}
pub fn member_leave(&self, user: u64) {
self.send_json(json!({"type": "member_leave", "user": user.to_string()}));
}
/// A message of a registered chat channel. `to`: the user Jarvis answered, on its own messages.
#[allow(clippy::too_many_arguments)]
pub fn chat_message(&self, channel: u64, message: u64, author: u64, name: &str, text: &str, ts: f64,
to: Option<u64>) {
let mut v = json!({"type": "chat_message", "channel": channel.to_string(), "message_id": message.to_string(),
"author": author.to_string(), "name": name, "text": text, "ts": ts});
if let Some(to) = to {
v["to"] = Value::String(to.to_string());
}
self.send_json(v);
}
pub fn chat_edit(&self, channel: u64, message: u64, text: &str) {
self.send_json(json!({"type": "chat_edit", "channel": channel.to_string(), "message_id": message.to_string(),
"text": text}));
}
/// The registered chat channels, their switches and the bot's own user ID, for the chat tool.
pub fn chat_channels(&self, bot: Option<u64>, channels: &[crate::settings::ChatChannelSettings]) {
let list: Vec<Value> = channels.iter().map(|c| json!({"id": c.id.to_string(), "name": c.name,
"description": c.description, "read": c.read, "send": c.send, "edit": c.edit, "react": c.react,
"delete": c.delete, "pin": c.pin})).collect();
let mut v = json!({"type": "chat_channels", "channels": list});
if let Some(b) = bot {
v["bot"] = Value::String(b.to_string());
}
self.send_json(v);
}
/// The outcome of a `chat_action`: the message ids it made or touched, or why it failed.
pub fn chat_action_result(&self, id: &str, result: Result<Vec<u64>, String>) {
let v = match result {
Ok(ids) => json!({"type": "chat_action_result", "id": id, "ok": true,
"message_ids": ids.iter().map(|i| i.to_string()).collect::<Vec<_>>()}),
Err(e) => json!({"type": "chat_action_result", "id": id, "ok": false, "error": e}),
};
self.send_json(v);
}
pub fn chat_delete(&self, channel: u64, message: u64) {
self.send_json(json!({"type": "chat_delete", "channel": channel.to_string(),
"message_id": message.to_string()}));
}
/// Change STT settings now and for every later connection.
pub fn configure(&self, changes: Map<String, Value>) {
self.inner.state.lock().unwrap().stt.extend(changes.clone());
let mut msg = changes;
msg.insert("type".into(), "config".into());
self.send_json(Value::Object(msg));
}
}
impl TtsBackend for WorkerClient {
/// Speak `text` as request `id`; with `more` the request stays open for more text. Returns false
/// (and the request ends as failed) when the worker is not connected.
fn tts(&self, id: &str, text: &str, more: bool) -> bool {
let fresh = self.inner.state.lock().unwrap().open_tts.insert(id.to_owned());
let sent = self.send_json(json!({"type": "tts", "id": id, "text": text, "more": more}));
if !sent {
let was_open = self.inner.state.lock().unwrap().open_tts.remove(id);
if fresh || was_open {
let _ = self.inner.events.send(WorkerEvent::TtsDone(TtsDone { id: id.into(), chunks: 0,
cancelled: false, failed: true }));
}
}
sent
}
fn tts_cancel(&self, id: &str) {
self.send_json(json!({"type": "tts_cancel", "id": id}));
}
}
fn mark_disconnected(inner: &Inner) {
let was = inner.ready.send_replace(None).is_some();
let (open, dropped) = {
let mut st = inner.state.lock().unwrap();
st.conn = None;
(std::mem::take(&mut st.open_tts), st.dropped_frames)
};
for id in open {
let _ = inner.events.send(WorkerEvent::TtsDone(TtsDone { id, chunks: 0, cancelled: false, failed: true }));
}
if was {
debug_log!("worker", "voice worker disconnected (dropped audio frames so far: {dropped})");
let _ = inner.events.send(WorkerEvent::Disconnected);
}
}
async fn run(inner: Arc<Inner>) {
let mut backoff = inner.opts.backoff;
loop {
if inner.state.lock().unwrap().closed {
return;
}
if connect_once(&inner).await {
backoff = inner.opts.backoff;
}
mark_disconnected(&inner);
if inner.state.lock().unwrap().closed {
return;
}
tokio::select! {
_ = tokio::time::sleep(backoff) => {}
_ = inner.wake.notified() => {}
}
backoff = (backoff * 2).min(inner.opts.max_backoff);
}
}
/// One connection's life. True if it got as far as hello_ok.
async fn connect_once(inner: &Arc<Inner>) -> bool {
let ws = match tokio_tungstenite::connect_async(inner.opts.url.as_str()).await {
Ok((ws, _)) => ws,
Err(e) => {
// Refused is normal while the worker is still loading its models.
debug_log!("worker", "voice worker connect failed: {e}");
return false;
}
};
let (mut write, mut read) = ws.split();
let stt = inner.state.lock().unwrap().stt.clone();
let o = &inner.opts;
let hello = json!({"type": "hello", "token": o.token,
"audio": {"rate": o.audio.rate, "channels": o.audio.channels},
"tts": {"rate": o.tts.rate, "channels": o.tts.channels}, "stt": stt});
if write.send(Message::text(hello.to_string())).await.is_err() {
return false;
}
let hello_ok = tokio::time::timeout(o.hello_timeout, async {
while let Some(Ok(msg)) = read.next().await {
match msg {
Message::Text(t) => {
let v: Value = serde_json::from_str(t.as_str()).ok()?;
if v["type"] == "hello_ok" {
return serde_json::from_value::<HelloOk>(v).ok();
}
}
Message::Close(Some(f)) => {
if matches!(f.code, CloseCode::Library(4400) | CloseCode::Library(4401)) {
error_log!("worker", "voice worker refused the connection ({} {})", u16::from(f.code), f.reason);
}
return None;
}
_ => {}
}
}
None
})
.await;
let Ok(Some(hello_ok)) = hello_ok else { return false };
let (tx, mut rx) = mpsc::unbounded_channel::<Message>();
{
let mut st = inner.state.lock().unwrap();
if st.closed {
return true;
}
st.conn = Some(tx);
}
debug_log!("worker", "voice worker ready stt={} tts={}", hello_ok.stt_ready, hello_ok.tts_ready);
inner.ready.send_replace(Some(hello_ok.clone()));
let _ = inner.events.send(WorkerEvent::Ready(hello_ok));
loop {
tokio::select! {
out = rx.recv() => match out {
Some(Message::Close(_)) | None => { let _ = write.close().await; break; }
Some(m) => if write.send(m).await.is_err() { break },
},
msg = read.next() => match msg {
Some(Ok(Message::Binary(b))) => on_binary(inner, &b),
Some(Ok(Message::Text(t))) => on_text(inner, t.as_str()),
Some(Ok(Message::Close(_))) | Some(Err(_)) | None => break,
Some(Ok(_)) => {}
},
}
}
true
}
fn on_binary(inner: &Inner, b: &[u8]) {
if b.first() != Some(&KIND_TTS) {
return;
}
if let Some(f) = decode_tts_frame(b) {
if inner.state.lock().unwrap().open_tts.contains(&f.id) {
let _ = inner.events.send(WorkerEvent::TtsAudio { id: f.id, seq: f.seq, pcm: f.pcm });
}
}
}
fn on_text(inner: &Inner, t: &str) {
let Ok(v) = serde_json::from_str::<Value>(t) else { return };
let kind = v["type"].as_str().unwrap_or_default().to_owned();
let ev = match kind.as_str() {
"speech_start" => serde_json::from_value(v).ok().map(WorkerEvent::SpeechStart),
"partial" => serde_json::from_value(v).ok().map(WorkerEvent::Partial),
"final" => serde_json::from_value(v).ok().map(WorkerEvent::Final),
"chat_action" => serde_json::from_value(v).ok().map(WorkerEvent::ChatAction),
"tts_done" => serde_json::from_value::<TtsDone>(v).ok().and_then(|d| {
inner.state.lock().unwrap().open_tts.remove(&d.id).then_some(WorkerEvent::TtsDone(d))
}),
"error" => serde_json::from_value::<WorkerError>(v).ok().map(|e| {
debug_log!("worker", "voice worker error {}", e.code);
// A request the worker refused outright never gets a tts_done.
if e.code == "tts_unavailable" {
if let Some(id) = &e.id {
if inner.state.lock().unwrap().open_tts.remove(id) {
let _ = inner.events.send(WorkerEvent::Error(e.clone()));
return WorkerEvent::TtsDone(TtsDone { id: id.clone(), chunks: 0, cancelled: false, failed: true });
}
}
}
WorkerEvent::Error(e)
}),
_ => None,
};
if let Some(ev) = ev {
let _ = inner.events.send(ev);
}
}