The share target checks that the person asked about shares their screen in Jarvis's voice channel, finds Discord's popped-out window (title not ending in Discord, the voice channel's name first) and, when there is none, asks for it in the channel and looks again every second for 30 s. Screen actions now wait as long as PC actions in the worker. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
821 lines
31 KiB
Rust
821 lines
31 KiB
Rust
//! WebSocket client for the Python voice worker (`src/voice_worker/voice_worker.spec.md`).
|
|
//!
|
|
//! Keeps one connection open, says hello with the app's token, and reconnects with backoff when the
|
|
//! worker restarts. While disconnected, audio is dropped (the worker would not know the speaker's
|
|
//! state anyway) and every open TTS request ends as failed, so playback never waits forever.
|
|
|
|
use std::collections::HashSet;
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::Duration;
|
|
|
|
use futures_util::{SinkExt, StreamExt};
|
|
use serde::Deserialize;
|
|
use serde_json::{json, Map, Value};
|
|
use tokio::sync::{mpsc, watch, Notify};
|
|
use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
|
|
use tokio_tungstenite::tungstenite::Message;
|
|
|
|
use crate::frames::{decode_tts_frame, encode_audio_frame, KIND_TTS};
|
|
use crate::playback::TtsBackend;
|
|
use crate::{debug_log, error_log};
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub struct PcmFormat {
|
|
pub rate: u32,
|
|
pub channels: u8,
|
|
}
|
|
|
|
/// Discord voice after Opus decoding, and what the worker is asked to send back.
|
|
pub const DISCORD_PCM: PcmFormat = PcmFormat { rate: 48000, channels: 2 };
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct WorkerClientOptions {
|
|
pub url: String,
|
|
pub token: String,
|
|
pub audio: PcmFormat,
|
|
pub tts: PcmFormat,
|
|
pub stt: Map<String, Value>,
|
|
/// The worker's memory settings, sent in `hello` (voice_worker.spec.md).
|
|
pub memory: Map<String, Value>,
|
|
/// First reconnect delay; doubles up to `max_backoff`.
|
|
pub backoff: Duration,
|
|
pub max_backoff: Duration,
|
|
pub hello_timeout: Duration,
|
|
}
|
|
|
|
impl WorkerClientOptions {
|
|
pub fn new(url: impl Into<String>, token: impl Into<String>) -> Self {
|
|
Self {
|
|
url: url.into(),
|
|
token: token.into(),
|
|
audio: DISCORD_PCM,
|
|
tts: DISCORD_PCM,
|
|
stt: Map::new(),
|
|
memory: Map::new(),
|
|
backoff: Duration::from_millis(500),
|
|
max_backoff: Duration::from_secs(5),
|
|
hello_timeout: Duration::from_secs(5),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, PartialEq)]
|
|
pub struct HelloOk {
|
|
pub stt_ready: bool,
|
|
pub tts_ready: bool,
|
|
#[serde(default)]
|
|
pub stt: Map<String, Value>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
|
pub struct SpeechStart {
|
|
pub speaker: String,
|
|
pub utt: u32,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
|
pub struct Partial {
|
|
pub speaker: String,
|
|
pub utt: u32,
|
|
pub text: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
|
pub struct Final {
|
|
pub speaker: String,
|
|
pub utt: u32,
|
|
pub text: String,
|
|
pub end_reason: String,
|
|
pub audio_ms: u64,
|
|
pub wait_ms: u64,
|
|
pub stt_ms: u64,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
|
pub struct TtsDone {
|
|
pub id: String,
|
|
pub chunks: u32,
|
|
pub cancelled: bool,
|
|
pub failed: bool,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
|
pub struct WorkerError {
|
|
pub code: String,
|
|
pub message: String,
|
|
#[serde(default)]
|
|
pub id: Option<String>,
|
|
}
|
|
|
|
/// The brain's chat channel tool asks the app to act (voice_worker.spec.md, chat_channels.spec.md).
|
|
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
|
pub struct ChatAction {
|
|
pub id: String,
|
|
/// `send`, `edit`, `react`, `delete`, `pin` or `unpin`.
|
|
pub action: String,
|
|
pub channel: String,
|
|
#[serde(default)]
|
|
pub message_id: Option<String>,
|
|
#[serde(default)]
|
|
pub text: Option<String>,
|
|
#[serde(default)]
|
|
pub emoji: Option<String>,
|
|
/// Who asked (for the activity log).
|
|
#[serde(default)]
|
|
pub user: Option<String>,
|
|
}
|
|
|
|
/// The brain's schedule tool asks the app for the person asking (voice_worker.spec.md, "Schedules").
|
|
#[derive(Debug, Clone, Deserialize, PartialEq)]
|
|
pub struct ScheduleAction {
|
|
pub id: String,
|
|
/// `add`, `list` or `delete`.
|
|
pub action: String,
|
|
pub user: String,
|
|
/// `voice` or `chat`: where they asked.
|
|
pub via: String,
|
|
#[serde(default)]
|
|
pub channel: Option<String>,
|
|
#[serde(default)]
|
|
pub kind: Option<String>,
|
|
#[serde(default)]
|
|
pub text: Option<String>,
|
|
#[serde(default)]
|
|
pub at: Option<f64>,
|
|
#[serde(default)]
|
|
pub repeat: Option<String>,
|
|
/// delete: the schedule's id.
|
|
#[serde(default)]
|
|
pub schedule: Option<String>,
|
|
}
|
|
|
|
/// The brain's personalNotes tool asks the app for the person asking (voice_worker.spec.md,
|
|
/// "Personal notes").
|
|
#[derive(Debug, Clone, Deserialize, PartialEq)]
|
|
pub struct NoteAction {
|
|
pub id: String,
|
|
/// `add`, `list` or `delete`.
|
|
pub action: String,
|
|
pub user: String,
|
|
#[serde(default)]
|
|
pub text: Option<String>,
|
|
/// delete: the note's id.
|
|
#[serde(default)]
|
|
pub note: Option<String>,
|
|
}
|
|
|
|
/// The brain's screenView tool asks the app for the person asking (voice_worker.spec.md, "Screen
|
|
/// viewing").
|
|
#[derive(Debug, Clone, Deserialize, PartialEq)]
|
|
pub struct ScreenAction {
|
|
pub id: String,
|
|
/// `capture` or `list`.
|
|
pub action: String,
|
|
pub user: String,
|
|
/// `voice` or `chat`.
|
|
#[serde(default)]
|
|
pub via: Option<String>,
|
|
/// capture: `screen`, `monitor`, `window` or `share`.
|
|
#[serde(default)]
|
|
pub target: Option<String>,
|
|
#[serde(default)]
|
|
pub monitor: Option<u32>,
|
|
#[serde(default)]
|
|
pub window: Option<String>,
|
|
/// share: the name of the person whose screen share is asked about.
|
|
#[serde(default)]
|
|
pub person: Option<String>,
|
|
/// `fast`, `normal` or `detailed`.
|
|
#[serde(default)]
|
|
pub detail: Option<String>,
|
|
/// share: the name of Jarvis's voice channel, set by the core (the popped-out window's title).
|
|
#[serde(skip)]
|
|
pub voice_channel: Option<String>,
|
|
}
|
|
|
|
/// The brain's pcControl tool asks the app for the person asking (voice_worker.spec.md, "PC control").
|
|
#[derive(Debug, Clone, Deserialize, PartialEq)]
|
|
pub struct PcAction {
|
|
pub id: String,
|
|
/// `windows`, `read`, `click`, `type`, `keys`, `run` or `close`.
|
|
pub action: String,
|
|
pub user: String,
|
|
/// `voice` or `chat`: where they asked.
|
|
#[serde(default)]
|
|
pub via: Option<String>,
|
|
#[serde(default)]
|
|
pub channel: Option<String>,
|
|
/// A window id from `windows`, or part of a program name or title.
|
|
#[serde(default)]
|
|
pub window: Option<String>,
|
|
/// An element from the latest reading (`@<snapshot>:e<n>`).
|
|
#[serde(default, rename = "ref")]
|
|
pub element: Option<String>,
|
|
#[serde(default)]
|
|
pub text: Option<String>,
|
|
#[serde(default)]
|
|
pub keys: Option<String>,
|
|
#[serde(default)]
|
|
pub program: Option<String>,
|
|
/// The person agreed in their next message (the tool checked it).
|
|
#[serde(default)]
|
|
pub consent: bool,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
pub enum WorkerEvent {
|
|
Ready(HelloOk),
|
|
Disconnected,
|
|
SpeechStart(SpeechStart),
|
|
Partial(Partial),
|
|
Final(Final),
|
|
TtsAudio { id: String, seq: u32, pcm: Vec<u8> },
|
|
TtsDone(TtsDone),
|
|
ChatAction(ChatAction),
|
|
ScheduleAction(ScheduleAction),
|
|
NoteAction(NoteAction),
|
|
ScreenAction(ScreenAction),
|
|
PcAction(PcAction),
|
|
/// The brain's answer to `reply` (empty and `failed` when it gave none).
|
|
ReplyText { id: String, text: String, failed: bool },
|
|
/// Was the utterance said to Jarvis? (a failed judgement is a no)
|
|
Addressed { id: String, addressed: bool },
|
|
/// Candidate ids, most important first (empty when the ranking failed).
|
|
Ranked { id: String, order: Vec<String> },
|
|
Error(WorkerError),
|
|
}
|
|
|
|
struct State {
|
|
conn: Option<mpsc::UnboundedSender<Message>>,
|
|
stt: Map<String, Value>,
|
|
/// TTS requests sent and not yet done, so a lost connection can end them.
|
|
open_tts: HashSet<String>,
|
|
closed: bool,
|
|
dropped_frames: u64,
|
|
}
|
|
|
|
struct Inner {
|
|
opts: WorkerClientOptions,
|
|
state: Mutex<State>,
|
|
events: mpsc::UnboundedSender<WorkerEvent>,
|
|
ready: watch::Sender<Option<HelloOk>>,
|
|
wake: Notify,
|
|
/// The last switch the brain reported idle for (`llm_idle` names it).
|
|
llm_idle: watch::Sender<u64>,
|
|
/// The LLM state the brain should have: sent again on every connection, so a worker that
|
|
/// missed it (a dropped connection, a restarted worker) catches up.
|
|
llm: Mutex<Option<Value>>,
|
|
/// The latest speech speed set while running, sent again on every connection: a restarted worker
|
|
/// starts with the speed of its arguments.
|
|
tts_speed: Mutex<Option<f64>>,
|
|
/// The number of the latest switch.
|
|
llm_switches: std::sync::atomic::AtomicU64,
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
pub struct WorkerClient {
|
|
inner: Arc<Inner>,
|
|
}
|
|
|
|
impl WorkerClient {
|
|
/// The client and the stream of worker events. Call `start()` to connect.
|
|
pub fn new(opts: WorkerClientOptions) -> (Self, mpsc::UnboundedReceiver<WorkerEvent>) {
|
|
let (events, rx) = mpsc::unbounded_channel();
|
|
let stt = opts.stt.clone();
|
|
let inner = Arc::new(Inner {
|
|
opts,
|
|
state: Mutex::new(State { conn: None, stt, open_tts: HashSet::new(), closed: false, dropped_frames: 0 }),
|
|
events,
|
|
ready: watch::channel(None).0,
|
|
wake: Notify::new(),
|
|
llm_idle: watch::channel(0).0,
|
|
llm: Mutex::new(None),
|
|
tts_speed: Mutex::new(None),
|
|
llm_switches: std::sync::atomic::AtomicU64::new(0),
|
|
});
|
|
(Self { inner }, rx)
|
|
}
|
|
|
|
/// Start connecting; reconnects until `close()`.
|
|
pub fn start(&self) {
|
|
let inner = self.inner.clone();
|
|
tokio::spawn(async move { run(inner).await });
|
|
}
|
|
|
|
/// The worker's `hello_ok` while connected (STT and TTS readiness), else None.
|
|
pub fn hello(&self) -> Option<HelloOk> {
|
|
self.inner.ready.borrow().clone()
|
|
}
|
|
|
|
pub fn is_ready(&self) -> bool {
|
|
self.inner.ready.borrow().is_some()
|
|
}
|
|
|
|
/// Resolve once the worker has said hello_ok, or None after `timeout`.
|
|
pub async fn wait_ready(&self, timeout: Duration) -> Option<HelloOk> {
|
|
let mut rx = self.inner.ready.subscribe();
|
|
tokio::time::timeout(timeout, async {
|
|
loop {
|
|
if let Some(h) = rx.borrow_and_update().clone() {
|
|
return h;
|
|
}
|
|
if rx.changed().await.is_err() {
|
|
std::future::pending::<()>().await;
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.ok()
|
|
}
|
|
|
|
pub fn close(&self) {
|
|
{
|
|
let mut st = self.inner.state.lock().unwrap();
|
|
st.closed = true;
|
|
if let Some(tx) = st.conn.take() {
|
|
let _ = tx.send(Message::Close(None));
|
|
}
|
|
}
|
|
self.inner.wake.notify_waiters();
|
|
self.inner.wake.notify_one();
|
|
mark_disconnected(&self.inner);
|
|
}
|
|
|
|
fn send(&self, msg: Message) -> bool {
|
|
let st = self.inner.state.lock().unwrap();
|
|
match &st.conn {
|
|
Some(tx) if self.inner.ready.borrow().is_some() => tx.send(msg).is_ok(),
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
fn send_json(&self, v: Value) -> bool {
|
|
self.send(Message::text(v.to_string()))
|
|
}
|
|
|
|
/// One frame of a speaker's PCM in the hello "audio" format. Dropped while disconnected.
|
|
pub fn send_audio(&self, speaker: &str, pcm: &[i16]) {
|
|
let Some(frame) = encode_audio_frame(speaker, pcm) else { return };
|
|
if !self.send(Message::binary(frame)) {
|
|
self.inner.state.lock().unwrap().dropped_frames += 1;
|
|
}
|
|
}
|
|
|
|
pub fn speaking_end(&self, speaker: &str) {
|
|
self.send_json(json!({"type": "speaking_end", "speaker": speaker}));
|
|
}
|
|
|
|
pub fn speaker_leave(&self, speaker: &str) {
|
|
self.send_json(json!({"type": "speaker_leave", "speaker": speaker}));
|
|
}
|
|
|
|
// --- history (voice_worker.spec.md "History and brain"). Dropped while disconnected; the
|
|
// discord core sends the voice state again when the worker is ready. ----------------------
|
|
|
|
/// Jarvis is in (or moved to) this voice channel.
|
|
pub fn voice_join(&self, channel: u64) {
|
|
self.send_json(json!({"type": "voice_join", "channel": channel.to_string()}));
|
|
}
|
|
|
|
pub fn voice_leave(&self) {
|
|
self.send_json(json!({"type": "voice_leave"}));
|
|
}
|
|
|
|
pub fn member_join(&self, user: u64, name: &str) {
|
|
self.send_json(json!({"type": "member_join", "user": user.to_string(), "name": name}));
|
|
}
|
|
|
|
pub fn member_leave(&self, user: u64) {
|
|
self.send_json(json!({"type": "member_leave", "user": user.to_string()}));
|
|
}
|
|
|
|
/// A message of a registered chat channel. `to`: the user Jarvis answered, on its own messages.
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub fn chat_message(&self, channel: u64, message: u64, author: u64, name: &str, text: &str, ts: f64,
|
|
to: Option<u64>) {
|
|
let mut v = json!({"type": "chat_message", "channel": channel.to_string(), "message_id": message.to_string(),
|
|
"author": author.to_string(), "name": name, "text": text, "ts": ts});
|
|
if let Some(to) = to {
|
|
v["to"] = Value::String(to.to_string());
|
|
}
|
|
self.send_json(v);
|
|
}
|
|
|
|
pub fn chat_edit(&self, channel: u64, message: u64, text: &str) {
|
|
self.send_json(json!({"type": "chat_edit", "channel": channel.to_string(), "message_id": message.to_string(),
|
|
"text": text}));
|
|
}
|
|
|
|
/// The registered chat channels, their switches and the bot's own user ID, for the chat tool.
|
|
pub fn chat_channels(&self, bot: Option<u64>, channels: &[crate::settings::ChatChannelSettings]) {
|
|
let list: Vec<Value> = channels.iter().map(|c| json!({"id": c.id.to_string(), "name": c.name,
|
|
"description": c.description, "read": c.read, "send": c.send, "edit": c.edit, "react": c.react,
|
|
"delete": c.delete, "pin": c.pin})).collect();
|
|
let mut v = json!({"type": "chat_channels", "channels": list});
|
|
if let Some(b) = bot {
|
|
v["bot"] = Value::String(b.to_string());
|
|
}
|
|
self.send_json(v);
|
|
}
|
|
|
|
/// The outcome of a `chat_action`: the message ids it made or touched, or why it failed.
|
|
pub fn chat_action_result(&self, id: &str, result: Result<Vec<u64>, String>) {
|
|
let v = match result {
|
|
Ok(ids) => json!({"type": "chat_action_result", "id": id, "ok": true,
|
|
"message_ids": ids.iter().map(|i| i.to_string()).collect::<Vec<_>>()}),
|
|
Err(e) => json!({"type": "chat_action_result", "id": id, "ok": false, "error": e}),
|
|
};
|
|
self.send_json(v);
|
|
}
|
|
|
|
/// The outcome of a `schedule_action`: `{item}` or `{items}`, or why it failed.
|
|
pub fn schedule_action_result(&self, id: &str, result: Result<Value, String>) {
|
|
self.app_action_result("schedule_action_result", id, result);
|
|
}
|
|
|
|
/// The outcome of a `note_action`: `{item}` or `{items}`, or why it failed.
|
|
pub fn note_action_result(&self, id: &str, result: Result<Value, String>) {
|
|
self.app_action_result("note_action_result", id, result);
|
|
}
|
|
|
|
/// The outcome of a `screen_action`: `{item, images}`, or why it failed.
|
|
pub fn screen_action_result(&self, id: &str, result: Result<Value, String>) {
|
|
self.app_action_result("screen_action_result", id, result);
|
|
}
|
|
|
|
/// The outcome of a `pc_action`: `{ok, item}`, `{ok: false, error}` or `{ok: false, ask, kinds,
|
|
/// window}`, as `pc.rs` made it.
|
|
pub fn pc_action_result(&self, id: &str, mut reply: Value) {
|
|
reply["type"] = json!("pc_action_result");
|
|
reply["id"] = json!(id);
|
|
self.send_json(reply);
|
|
}
|
|
|
|
fn app_action_result(&self, kind: &str, id: &str, result: Result<Value, String>) {
|
|
let msg = match result {
|
|
Ok(Value::Object(mut m)) => {
|
|
m.insert("type".into(), json!(kind));
|
|
m.insert("id".into(), json!(id));
|
|
m.insert("ok".into(), json!(true));
|
|
Value::Object(m)
|
|
}
|
|
Ok(_) => json!({"type": kind, "id": id, "ok": true}),
|
|
Err(e) => json!({"type": kind, "id": id, "ok": false, "error": e}),
|
|
};
|
|
self.send_json(msg);
|
|
}
|
|
|
|
pub fn chat_delete(&self, channel: u64, message: u64) {
|
|
self.send_json(json!({"type": "chat_delete", "channel": channel.to_string(),
|
|
"message_id": message.to_string()}));
|
|
}
|
|
|
|
// --- brain (voice_worker.spec.md "History and brain", conversation.spec.md) -------------------
|
|
|
|
/// Ask the brain to answer `user`. With `speak` the answer is spoken under `id` (its audio and
|
|
/// `tts_done` come like a TTS request's); it ends as failed when it cannot be sent, the brain gives
|
|
/// no answer, or the worker refuses it.
|
|
/// `via`: `voice`, `chat` or `task` (a scheduled task: read-only tools, never spoken).
|
|
pub fn reply(&self, id: &str, user: u64, text: &str, via: &str, speak: bool) -> bool {
|
|
self.reply_in(id, user, text, via, speak, None, &[])
|
|
}
|
|
|
|
/// `reply`, naming the chat channel a chat message was written in (the brain's schedule tool
|
|
/// delivers there) and carrying the asker's standing instructions (`notes`, at most 20).
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub fn reply_in(&self, id: &str, user: u64, text: &str, via: &str, speak: bool, channel: Option<u64>,
|
|
notes: &[String]) -> bool {
|
|
if speak {
|
|
self.inner.state.lock().unwrap().open_tts.insert(id.to_owned());
|
|
}
|
|
let mut msg = json!({"type": "reply", "id": id, "user": user.to_string(), "text": text, "via": via,
|
|
"speak": speak});
|
|
if let Some(c) = channel {
|
|
msg["channel"] = json!(c.to_string());
|
|
}
|
|
if !notes.is_empty() {
|
|
msg["notes"] = json!(notes);
|
|
}
|
|
let sent = self.send_json(msg);
|
|
if !sent && speak {
|
|
self.end_failed(id);
|
|
}
|
|
sent
|
|
}
|
|
|
|
/// Whether the chat answer `id` was posted (voice_worker.spec.md "History and brain"): a
|
|
/// question in an answer the person never saw cannot be confirmed in their next message.
|
|
pub fn reply_delivered(&self, id: &str, ok: bool) {
|
|
self.send_json(json!({"type": "reply_delivered", "id": id, "ok": ok}));
|
|
}
|
|
|
|
fn end_failed(&self, id: &str) {
|
|
if self.inner.state.lock().unwrap().open_tts.remove(id) {
|
|
let _ = self.inner.events.send(WorkerEvent::TtsDone(TtsDone { id: id.into(), chunks: 0, cancelled: false,
|
|
failed: true }));
|
|
}
|
|
}
|
|
|
|
/// Ask whether `text` was said to Jarvis (call mode `addressed`).
|
|
pub fn addressed(&self, id: &str, user: u64, text: &str, people: usize, callers: usize) -> bool {
|
|
self.send_json(json!({"type": "addressed", "id": id, "user": user.to_string(), "text": text,
|
|
"people": people, "callers": callers}))
|
|
}
|
|
|
|
/// Tell the brain the LLM switches (voice_worker.spec.md, "LLM switches"): it starts no new
|
|
/// LLM work, says `notice` once to the first spoken request that waits, and reports idle for
|
|
/// this switch, whose number is returned for `wait_llm_idle`.
|
|
pub fn llm_switching(&self, notice: Option<&str>) -> u64 {
|
|
let id = self.inner.llm_switches.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1;
|
|
let mut m = json!({"type": "llm", "state": "switching", "id": id});
|
|
if let Some(n) = notice {
|
|
m["notice"] = json!(n);
|
|
}
|
|
// Kept and sent under one lock, so a connection being set up sends this or a later state.
|
|
let mut llm = self.inner.llm.lock().unwrap();
|
|
*llm = Some(m.clone());
|
|
self.send_json(m);
|
|
id
|
|
}
|
|
|
|
/// Tell the brain the LLM is `ready` (answer what waited) or `paused` (answer nothing).
|
|
pub fn llm_state(&self, state: &str) -> bool {
|
|
self.set_llm(json!({"type": "llm", "state": state}))
|
|
}
|
|
|
|
/// Tell the brain the LLM is ready, and whether a game runs (then partial passes wait while an
|
|
/// answer is written).
|
|
pub fn llm_ready(&self, game: bool) -> bool {
|
|
self.set_llm(json!({"type": "llm", "state": "ready", "game": game}))
|
|
}
|
|
|
|
/// Speak at `speed` from the next sentence on (`tts_speed`). False when not connected; it still
|
|
/// goes out on the next connection.
|
|
pub fn set_tts_speed(&self, speed: f64) -> bool {
|
|
let mut s = self.inner.tts_speed.lock().unwrap();
|
|
*s = Some(speed);
|
|
self.send_json(json!({"type": "tts_speed", "speed": speed}))
|
|
}
|
|
|
|
fn set_llm(&self, m: Value) -> bool {
|
|
let mut llm = self.inner.llm.lock().unwrap();
|
|
*llm = Some(m.clone());
|
|
self.send_json(m)
|
|
}
|
|
|
|
/// Wait until the brain reports idle for switch `id` (also when it came before the wait); a
|
|
/// report for an earlier switch does not count. At most `t`.
|
|
pub async fn wait_llm_idle(&self, id: u64, t: Duration) -> bool {
|
|
let mut rx = self.inner.llm_idle.subscribe();
|
|
tokio::time::timeout(t, rx.wait_for(|got| *got >= id)).await.is_ok_and(|r| r.is_ok())
|
|
}
|
|
|
|
/// Tell the brain a call is being said (a partial transcript), so it gets ready to answer it.
|
|
pub fn prepare(&self, user: u64, text: &str) -> bool {
|
|
self.send_json(json!({"type": "prepare", "user": user.to_string(), "text": text}))
|
|
}
|
|
|
|
/// Ask the brain to rank tied candidates `(id, user, text)` by importance.
|
|
pub fn rank(&self, id: &str, candidates: &[(String, u64, String)]) -> bool {
|
|
let list: Vec<Value> = candidates.iter()
|
|
.map(|(cid, user, text)| json!({"id": cid, "user": user.to_string(), "text": text}))
|
|
.collect();
|
|
self.send_json(json!({"type": "rank", "id": id, "candidates": list}))
|
|
}
|
|
|
|
/// Change STT settings now and for every later connection.
|
|
pub fn configure(&self, changes: Map<String, Value>) {
|
|
self.inner.state.lock().unwrap().stt.extend(changes.clone());
|
|
let mut msg = changes;
|
|
msg.insert("type".into(), "config".into());
|
|
self.send_json(Value::Object(msg));
|
|
}
|
|
}
|
|
|
|
impl TtsBackend for WorkerClient {
|
|
/// Speak `text` as request `id`; with `more` the request stays open for more text. Returns false
|
|
/// (and the request ends as failed) when the worker is not connected.
|
|
fn tts(&self, id: &str, text: &str, more: bool) -> bool {
|
|
let fresh = self.inner.state.lock().unwrap().open_tts.insert(id.to_owned());
|
|
let sent = self.send_json(json!({"type": "tts", "id": id, "text": text, "more": more}));
|
|
if !sent {
|
|
let was_open = self.inner.state.lock().unwrap().open_tts.remove(id);
|
|
if fresh || was_open {
|
|
let _ = self.inner.events.send(WorkerEvent::TtsDone(TtsDone { id: id.into(), chunks: 0,
|
|
cancelled: false, failed: true }));
|
|
}
|
|
}
|
|
sent
|
|
}
|
|
|
|
fn tts_cancel(&self, id: &str) {
|
|
self.send_json(json!({"type": "tts_cancel", "id": id}));
|
|
}
|
|
|
|
fn tts_played(&self, id: &str, played: u32, cancelled: bool, last_ms: Option<u32>) {
|
|
let mut m = json!({"type": "tts_played", "id": id, "played": played, "cancelled": cancelled});
|
|
if let Some(ms) = last_ms {
|
|
m["last_ms"] = json!(ms);
|
|
}
|
|
self.send_json(m);
|
|
}
|
|
}
|
|
|
|
fn mark_disconnected(inner: &Inner) {
|
|
let was = inner.ready.send_replace(None).is_some();
|
|
let (open, dropped) = {
|
|
let mut st = inner.state.lock().unwrap();
|
|
st.conn = None;
|
|
(std::mem::take(&mut st.open_tts), st.dropped_frames)
|
|
};
|
|
for id in open {
|
|
let _ = inner.events.send(WorkerEvent::TtsDone(TtsDone { id, chunks: 0, cancelled: false, failed: true }));
|
|
}
|
|
if was {
|
|
debug_log!("worker", "voice worker disconnected (dropped audio frames so far: {dropped})");
|
|
let _ = inner.events.send(WorkerEvent::Disconnected);
|
|
}
|
|
}
|
|
|
|
async fn run(inner: Arc<Inner>) {
|
|
let mut backoff = inner.opts.backoff;
|
|
loop {
|
|
if inner.state.lock().unwrap().closed {
|
|
return;
|
|
}
|
|
if connect_once(&inner).await {
|
|
backoff = inner.opts.backoff;
|
|
}
|
|
mark_disconnected(&inner);
|
|
if inner.state.lock().unwrap().closed {
|
|
return;
|
|
}
|
|
tokio::select! {
|
|
_ = tokio::time::sleep(backoff) => {}
|
|
_ = inner.wake.notified() => {}
|
|
}
|
|
backoff = (backoff * 2).min(inner.opts.max_backoff);
|
|
}
|
|
}
|
|
|
|
/// One connection's life. True if it got as far as hello_ok.
|
|
async fn connect_once(inner: &Arc<Inner>) -> bool {
|
|
let ws = match tokio_tungstenite::connect_async(inner.opts.url.as_str()).await {
|
|
Ok((ws, _)) => ws,
|
|
Err(e) => {
|
|
// Refused is normal while the worker is still loading its models.
|
|
debug_log!("worker", "voice worker connect failed: {e}");
|
|
return false;
|
|
}
|
|
};
|
|
let (mut write, mut read) = ws.split();
|
|
let stt = inner.state.lock().unwrap().stt.clone();
|
|
let o = &inner.opts;
|
|
let mut hello = json!({"type": "hello", "token": o.token,
|
|
"audio": {"rate": o.audio.rate, "channels": o.audio.channels},
|
|
"tts": {"rate": o.tts.rate, "channels": o.tts.channels}, "stt": stt});
|
|
if !o.memory.is_empty() {
|
|
hello["memory"] = Value::Object(o.memory.clone());
|
|
}
|
|
if write.send(Message::text(hello.to_string())).await.is_err() {
|
|
return false;
|
|
}
|
|
let hello_ok = tokio::time::timeout(o.hello_timeout, async {
|
|
while let Some(Ok(msg)) = read.next().await {
|
|
match msg {
|
|
Message::Text(t) => {
|
|
let v: Value = serde_json::from_str(t.as_str()).ok()?;
|
|
if v["type"] == "hello_ok" {
|
|
return serde_json::from_value::<HelloOk>(v).ok();
|
|
}
|
|
}
|
|
Message::Close(Some(f)) => {
|
|
if matches!(f.code, CloseCode::Library(4400) | CloseCode::Library(4401)) {
|
|
error_log!("worker", "voice worker refused the connection ({} {})", u16::from(f.code), f.reason);
|
|
}
|
|
return None;
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
None
|
|
})
|
|
.await;
|
|
let Ok(Some(hello_ok)) = hello_ok else { return false };
|
|
|
|
let (tx, mut rx) = mpsc::unbounded_channel::<Message>();
|
|
{
|
|
// Locks in the order llm, tts_speed, then state (as `llm_state` and `set_tts_speed` take
|
|
// them): the latest LLM state and speed go first, and the connection is ready for sending
|
|
// before one set meanwhile is sent.
|
|
let llm = inner.llm.lock().unwrap();
|
|
let speed = inner.tts_speed.lock().unwrap();
|
|
let mut st = inner.state.lock().unwrap();
|
|
if st.closed {
|
|
return true;
|
|
}
|
|
if let Some(m) = llm.clone() {
|
|
let _ = tx.send(Message::text(m.to_string()));
|
|
}
|
|
if let Some(s) = *speed {
|
|
let _ = tx.send(Message::text(json!({"type": "tts_speed", "speed": s}).to_string()));
|
|
}
|
|
st.conn = Some(tx);
|
|
inner.ready.send_replace(Some(hello_ok.clone()));
|
|
}
|
|
debug_log!("worker", "voice worker ready stt={} tts={}", hello_ok.stt_ready, hello_ok.tts_ready);
|
|
let _ = inner.events.send(WorkerEvent::Ready(hello_ok));
|
|
|
|
loop {
|
|
tokio::select! {
|
|
out = rx.recv() => match out {
|
|
Some(Message::Close(_)) | None => { let _ = write.close().await; break; }
|
|
Some(m) => if write.send(m).await.is_err() { break },
|
|
},
|
|
msg = read.next() => match msg {
|
|
Some(Ok(Message::Binary(b))) => on_binary(inner, &b),
|
|
Some(Ok(Message::Text(t))) => on_text(inner, t.as_str()),
|
|
Some(Ok(Message::Close(_))) | Some(Err(_)) | None => break,
|
|
Some(Ok(_)) => {}
|
|
},
|
|
}
|
|
}
|
|
true
|
|
}
|
|
|
|
fn on_binary(inner: &Inner, b: &[u8]) {
|
|
if b.first() != Some(&KIND_TTS) {
|
|
return;
|
|
}
|
|
if let Some(f) = decode_tts_frame(b) {
|
|
if inner.state.lock().unwrap().open_tts.contains(&f.id) {
|
|
let _ = inner.events.send(WorkerEvent::TtsAudio { id: f.id, seq: f.seq, pcm: f.pcm });
|
|
}
|
|
}
|
|
}
|
|
|
|
fn on_text(inner: &Inner, t: &str) {
|
|
let Ok(v) = serde_json::from_str::<Value>(t) else { return };
|
|
let kind = v["type"].as_str().unwrap_or_default().to_owned();
|
|
let ev = match kind.as_str() {
|
|
"speech_start" => serde_json::from_value(v).ok().map(WorkerEvent::SpeechStart),
|
|
"partial" => serde_json::from_value(v).ok().map(WorkerEvent::Partial),
|
|
"final" => serde_json::from_value(v).ok().map(WorkerEvent::Final),
|
|
"chat_action" => serde_json::from_value(v).ok().map(WorkerEvent::ChatAction),
|
|
"schedule_action" => serde_json::from_value(v).ok().map(WorkerEvent::ScheduleAction),
|
|
"note_action" => serde_json::from_value(v).ok().map(WorkerEvent::NoteAction),
|
|
"screen_action" => serde_json::from_value(v).ok().map(WorkerEvent::ScreenAction),
|
|
"pc_action" => serde_json::from_value(v).ok().map(WorkerEvent::PcAction),
|
|
"llm_idle" => {
|
|
if let Some(id) = v["id"].as_u64() {
|
|
inner.llm_idle.send_if_modified(|got| if id > *got { *got = id; true } else { false });
|
|
}
|
|
return;
|
|
}
|
|
"reply_text" => {
|
|
let id = v["id"].as_str().unwrap_or_default().to_owned();
|
|
let failed = v["failed"].as_bool().unwrap_or(true);
|
|
let text = v["text"].as_str().unwrap_or_default().to_owned();
|
|
// No answer means nothing will be spoken: end the speech the app is waiting for.
|
|
if failed && inner.state.lock().unwrap().open_tts.remove(&id) {
|
|
let _ = inner.events.send(WorkerEvent::ReplyText { id: id.clone(), text, failed });
|
|
let _ = inner.events.send(WorkerEvent::TtsDone(TtsDone { id, chunks: 0, cancelled: false, failed: true }));
|
|
return;
|
|
}
|
|
Some(WorkerEvent::ReplyText { id, text, failed })
|
|
}
|
|
"addressed_result" => Some(WorkerEvent::Addressed {
|
|
id: v["id"].as_str().unwrap_or_default().to_owned(),
|
|
addressed: v["addressed"].as_bool().unwrap_or(false) && !v["failed"].as_bool().unwrap_or(false),
|
|
}),
|
|
"rank_result" => Some(WorkerEvent::Ranked {
|
|
id: v["id"].as_str().unwrap_or_default().to_owned(),
|
|
order: if v["failed"].as_bool().unwrap_or(false) { vec![] } else {
|
|
v["order"].as_array().map(|a| a.iter().filter_map(|x| x.as_str().map(str::to_owned)).collect())
|
|
.unwrap_or_default()
|
|
},
|
|
}),
|
|
"tts_done" => serde_json::from_value::<TtsDone>(v).ok().and_then(|d| {
|
|
inner.state.lock().unwrap().open_tts.remove(&d.id).then_some(WorkerEvent::TtsDone(d))
|
|
}),
|
|
"error" => serde_json::from_value::<WorkerError>(v).ok().map(|e| {
|
|
// Always logged: the log screen and its error report must show what the worker refused.
|
|
error_log!("worker", "voice worker error {} id={}", e.code, e.id.as_deref().unwrap_or("-"));
|
|
// A request the worker refused outright never gets a tts_done.
|
|
if e.code == "tts_unavailable" || e.code == "brain_unavailable" {
|
|
if let Some(id) = &e.id {
|
|
if inner.state.lock().unwrap().open_tts.remove(id) {
|
|
let _ = inner.events.send(WorkerEvent::Error(e.clone()));
|
|
return WorkerEvent::TtsDone(TtsDone { id: id.clone(), chunks: 0, cancelled: false, failed: true });
|
|
}
|
|
}
|
|
}
|
|
WorkerEvent::Error(e)
|
|
}),
|
|
_ => None,
|
|
};
|
|
if let Some(ev) = ev {
|
|
let _ = inner.events.send(ev);
|
|
}
|
|
}
|