Worker (voice_worker.spec.md "History and brain"): - --history <file> keeps the conversation history store; --brain hosts the jarvis reply engine (JarvisBrain, settings point ollama_base_url at /v1). - voice_join / voice_leave / member_join / member_leave keep presence; the session end closes it. Non-empty finals are stored under the speaker's name in the current voice channel. - chat_message / chat_edit / chat_delete keep registered chat channels. - reply builds the asker's scoped history context, runs the brain on a thread (one at a time), answers reply_text, stores a voice answer, and with speak=true says it under the same id. memory settings in hello and memory_config. brain_unavailable without --brain. App (discord_core.spec.md "History reporting"): - WorkerClient sends the new messages; the core reports the voice channel and its heard members after a join, a move and every worker reconnect, member joins and leaves, voice_leave when a session ends, and every message of a readable registered channel (its own too, with `to` on replies), edits and deletes. - live_discord runs the worker with --history and checks the store at the end: 8 finals under the speaker's name, the edited chat message only, Jarvis's own messages, 5 presence intervals, none left open (passed, 107.8 s). Eval: history recall gains a chat-record case and fails a reply that reads a record stamp out (field run with qwen2.5:7b did once in four). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
262 lines
11 KiB
Rust
262 lines
11 KiB
Rust
//! The worker client against a stand-in worker server (hello, events, reconnect, failed TTS).
|
|
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::Duration;
|
|
|
|
use futures_util::{SinkExt, StreamExt};
|
|
use jarvis_app::frames::{encode_audio_frame, KIND_TTS};
|
|
use jarvis_app::playback::TtsBackend;
|
|
use jarvis_app::worker_client::{TtsDone, WorkerClient, WorkerClientOptions, WorkerEvent};
|
|
use serde_json::{json, Value};
|
|
use tokio::net::TcpListener;
|
|
use tokio::sync::mpsc;
|
|
use tokio_tungstenite::tungstenite::Message;
|
|
|
|
const TOKEN: &str = "t0ken";
|
|
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
enum Got {
|
|
Json(Value),
|
|
Bin(Vec<u8>),
|
|
}
|
|
|
|
/// A fake worker: checks the hello, records what arrives, and lets the test push frames.
|
|
struct FakeWorker {
|
|
port: u16,
|
|
received: Arc<Mutex<Vec<Got>>>,
|
|
push: Arc<Mutex<Option<mpsc::UnboundedSender<Message>>>>,
|
|
task: tokio::task::JoinHandle<()>,
|
|
conns: Arc<Mutex<Vec<tokio::task::JoinHandle<()>>>>,
|
|
}
|
|
|
|
impl FakeWorker {
|
|
async fn start(port: u16) -> Self {
|
|
let listener = TcpListener::bind(("127.0.0.1", port)).await.unwrap();
|
|
let port = listener.local_addr().unwrap().port();
|
|
let received = Arc::new(Mutex::new(vec![]));
|
|
let push = Arc::new(Mutex::new(None));
|
|
let conns = Arc::new(Mutex::new(vec![]));
|
|
let (r, p, cs) = (received.clone(), push.clone(), conns.clone());
|
|
let task = tokio::spawn(async move {
|
|
while let Ok((tcp, _)) = listener.accept().await {
|
|
let ws = tokio_tungstenite::accept_async(tcp).await.unwrap();
|
|
let (mut tx, mut rx) = ws.split();
|
|
let (ptx, mut prx) = mpsc::unbounded_channel::<Message>();
|
|
*p.lock().unwrap() = Some(ptx.clone());
|
|
let r = r.clone();
|
|
cs.lock().unwrap().push(tokio::spawn(async move {
|
|
loop {
|
|
tokio::select! {
|
|
m = prx.recv() => match m { Some(m) => { if tx.send(m).await.is_err() { break } } None => break },
|
|
m = rx.next() => match m {
|
|
Some(Ok(Message::Text(t))) => {
|
|
let v: Value = serde_json::from_str(t.as_str()).unwrap();
|
|
if v["type"] == "hello" {
|
|
if v["token"] != TOKEN { let _ = tx.close().await; break; }
|
|
let ok = json!({"type": "hello_ok", "stt_ready": true, "tts_ready": true, "stt": v["stt"]});
|
|
let _ = ptx.send(Message::text(ok.to_string()));
|
|
}
|
|
r.lock().unwrap().push(Got::Json(v));
|
|
}
|
|
Some(Ok(Message::Binary(b))) => r.lock().unwrap().push(Got::Bin(b.to_vec())),
|
|
Some(Ok(_)) => {}
|
|
_ => break,
|
|
},
|
|
}
|
|
}
|
|
}));
|
|
}
|
|
});
|
|
Self { port, received, push, task, conns }
|
|
}
|
|
|
|
fn url(&self) -> String {
|
|
format!("ws://127.0.0.1:{}", self.port)
|
|
}
|
|
|
|
fn got(&self) -> Vec<Got> {
|
|
self.received.lock().unwrap().clone()
|
|
}
|
|
|
|
fn send(&self, m: Message) {
|
|
self.push.lock().unwrap().as_ref().unwrap().send(m).unwrap();
|
|
}
|
|
|
|
/// The worker dies: stop listening and drop the connection.
|
|
fn kill(self) -> u16 {
|
|
self.task.abort();
|
|
for c in self.conns.lock().unwrap().drain(..) {
|
|
c.abort();
|
|
}
|
|
*self.push.lock().unwrap() = None;
|
|
self.port
|
|
}
|
|
}
|
|
|
|
fn client(url: &str, stt: Value) -> (WorkerClient, mpsc::UnboundedReceiver<WorkerEvent>) {
|
|
let mut o = WorkerClientOptions::new(url, TOKEN);
|
|
o.stt = stt.as_object().cloned().unwrap_or_default();
|
|
o.backoff = Duration::from_millis(20);
|
|
o.max_backoff = Duration::from_millis(50);
|
|
WorkerClient::new(o)
|
|
}
|
|
|
|
async fn until(mut pred: impl FnMut() -> bool) {
|
|
tokio::time::timeout(Duration::from_secs(3), async {
|
|
while !pred() {
|
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
|
}
|
|
})
|
|
.await
|
|
.expect("timed out");
|
|
}
|
|
|
|
async fn next_matching(rx: &mut mpsc::UnboundedReceiver<WorkerEvent>, f: impl Fn(&WorkerEvent) -> bool) -> WorkerEvent {
|
|
tokio::time::timeout(Duration::from_secs(3), async {
|
|
loop {
|
|
let e = rx.recv().await.expect("events closed");
|
|
if f(&e) {
|
|
return e;
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.expect("timed out")
|
|
}
|
|
|
|
fn tts_frame(id: &str, seq: u32) -> Message {
|
|
let mut f = vec![KIND_TTS, id.len() as u8];
|
|
f.extend_from_slice(id.as_bytes());
|
|
f.extend_from_slice(&seq.to_le_bytes());
|
|
f.extend_from_slice(&[0; 8]);
|
|
Message::binary(f)
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn hello_carries_token_formats_and_stt_then_audio_flows() {
|
|
let w = FakeWorker::start(0).await;
|
|
let (c, _rx) = client(&w.url(), json!({"hotwords": ["자비스"]}));
|
|
c.start();
|
|
let ok = c.wait_ready(Duration::from_secs(2)).await.expect("ready");
|
|
assert_eq!(Value::Object(ok.stt), json!({"hotwords": ["자비스"]}));
|
|
let pcm = vec![1i16; 1920];
|
|
c.send_audio("123", &pcm);
|
|
c.speaking_end("123");
|
|
c.speaker_leave("123");
|
|
until(|| w.got().len() >= 4).await;
|
|
let got = w.got();
|
|
assert_eq!(got[0], Got::Json(json!({"type": "hello", "token": TOKEN, "audio": {"rate": 48000, "channels": 2},
|
|
"tts": {"rate": 48000, "channels": 2}, "stt": {"hotwords": ["자비스"]}})));
|
|
assert_eq!(got[1], Got::Bin(encode_audio_frame("123", &pcm).unwrap()));
|
|
assert_eq!(got[2..], [Got::Json(json!({"type": "speaking_end", "speaker": "123"})),
|
|
Got::Json(json!({"type": "speaker_leave", "speaker": "123"}))]);
|
|
c.close();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn events_pass_on_and_only_tts_audio_of_open_requests() {
|
|
let w = FakeWorker::start(0).await;
|
|
let (c, mut rx) = client(&w.url(), json!({}));
|
|
c.start();
|
|
c.wait_ready(Duration::from_secs(2)).await.unwrap();
|
|
assert!(c.tts("r1", "안녕하세요.", false));
|
|
until(|| w.got().iter().any(|g| matches!(g, Got::Json(v) if v["type"] == "tts"))).await;
|
|
w.send(tts_frame("zz", 0)); // not ours
|
|
w.send(tts_frame("r1", 0));
|
|
w.send(Message::text(json!({"type": "final", "speaker": "1", "utt": 1, "text": "네", "end_reason": "silence",
|
|
"audio_ms": 900, "wait_ms": 100, "stt_ms": 80}).to_string()));
|
|
w.send(Message::text(json!({"type": "tts_done", "id": "r1", "chunks": 1, "cancelled": false, "failed": false}).to_string()));
|
|
let mut seen = vec![];
|
|
while seen.len() < 4 {
|
|
let e = next_matching(&mut rx, |_| true).await;
|
|
if !matches!(e, WorkerEvent::Ready(_)) {
|
|
seen.push(e);
|
|
}
|
|
if matches!(seen.last(), Some(WorkerEvent::TtsDone(_))) {
|
|
break;
|
|
}
|
|
}
|
|
assert!(matches!(&seen[0], WorkerEvent::TtsAudio { id, seq: 0, .. } if id == "r1"));
|
|
assert!(matches!(&seen[1], WorkerEvent::Final(f) if f.text == "네" && f.end_reason == "silence"));
|
|
assert_eq!(seen[2], WorkerEvent::TtsDone(TtsDone { id: "r1".into(), chunks: 1, cancelled: false, failed: false }));
|
|
c.close();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_refused_tts_request_ends_as_failed() {
|
|
let w = FakeWorker::start(0).await;
|
|
let (c, mut rx) = client(&w.url(), json!({}));
|
|
c.start();
|
|
c.wait_ready(Duration::from_secs(2)).await.unwrap();
|
|
c.tts("r9", "x", false);
|
|
until(|| w.got().len() >= 2).await;
|
|
w.send(Message::text(json!({"type": "error", "code": "tts_unavailable", "message": "no", "id": "r9"}).to_string()));
|
|
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::TtsDone(_))).await;
|
|
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r9".into(), chunks: 0, cancelled: false, failed: true }));
|
|
c.close();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn reconnects_after_a_restart_and_fails_the_tts_in_flight() {
|
|
let w = FakeWorker::start(0).await;
|
|
let (c, mut rx) = client(&w.url(), json!({"language": "ko"}));
|
|
c.start();
|
|
c.wait_ready(Duration::from_secs(2)).await.unwrap();
|
|
c.tts("r1", "길게 말하는 중", false);
|
|
c.configure(json!({"partials": false}).as_object().unwrap().clone());
|
|
until(|| w.got().len() >= 3).await;
|
|
assert_eq!(w.got()[2], Got::Json(json!({"type": "config", "partials": false})));
|
|
let port = w.kill();
|
|
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::TtsDone(_))).await;
|
|
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r1".into(), chunks: 0, cancelled: false, failed: true }));
|
|
next_matching(&mut rx, |e| *e == WorkerEvent::Disconnected).await;
|
|
assert!(!c.is_ready());
|
|
c.send_audio("1", &[0, 0]); // dropped, not queued
|
|
let w = FakeWorker::start(port).await;
|
|
next_matching(&mut rx, |e| matches!(e, WorkerEvent::Ready(_))).await;
|
|
let got = w.got();
|
|
// The new hello carries the settings changed since, and no stale audio was queued.
|
|
assert!(matches!(&got[0], Got::Json(v) if v["stt"] == json!({"language": "ko", "partials": false})));
|
|
assert!(!got.iter().any(|g| matches!(g, Got::Bin(_))));
|
|
c.close();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn tts_while_disconnected_fails_at_once() {
|
|
let (c, mut rx) = client("ws://127.0.0.1:1", json!({}));
|
|
assert!(!c.tts("r1", "x", false));
|
|
let e = next_matching(&mut rx, |_| true).await;
|
|
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r1".into(), chunks: 0, cancelled: false, failed: true }));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn history_messages_follow_the_worker_protocol() {
|
|
let w = FakeWorker::start(0).await;
|
|
let (c, _rx) = client(&w.url(), json!({}));
|
|
c.start();
|
|
c.wait_ready(Duration::from_secs(2)).await.expect("ready");
|
|
c.voice_join(42);
|
|
c.member_join(7, "Alice");
|
|
c.member_leave(7);
|
|
c.chat_message(9, 100, 7, "Alice", "hi there", 1_800_000_000.5, None);
|
|
c.chat_message(9, 101, 1, "Jarvis", "hello", 1_800_000_001.0, Some(7));
|
|
c.chat_edit(9, 100, "hi, edited");
|
|
c.chat_delete(9, 100);
|
|
c.voice_leave();
|
|
until(|| w.got().len() >= 9).await;
|
|
let got: Vec<Value> = w.got()[1..].iter().map(|g| match g { Got::Json(v) => v.clone(), Got::Bin(_) => json!(null) }).collect();
|
|
assert_eq!(got, vec![
|
|
json!({"type": "voice_join", "channel": "42"}),
|
|
json!({"type": "member_join", "user": "7", "name": "Alice"}),
|
|
json!({"type": "member_leave", "user": "7"}),
|
|
json!({"type": "chat_message", "channel": "9", "message_id": "100", "author": "7", "name": "Alice",
|
|
"text": "hi there", "ts": 1_800_000_000.5}),
|
|
json!({"type": "chat_message", "channel": "9", "message_id": "101", "author": "1", "name": "Jarvis",
|
|
"text": "hello", "ts": 1_800_000_001.0, "to": "7"}),
|
|
json!({"type": "chat_edit", "channel": "9", "message_id": "100", "text": "hi, edited"}),
|
|
json!({"type": "chat_delete", "channel": "9", "message_id": "100"}),
|
|
json!({"type": "voice_leave"}),
|
|
]);
|
|
c.close();
|
|
}
|