Files
javis_bot/app/tests/worker_client.rs
javis-bot 7e096b449e feat(app): performance stages move the LLM between the normal and the game placement
- while the services run, game mode decides the stage (perf.rs): a game means the game stage at
  once (or pause with game.pauseInGame); its end brings the normal stage back after
  game.revertSeconds (default 30)
- a change tells the voice worker the LLM switches (llm switching, with a notice), waits until
  the brain has written the answers in progress (llm_idle, at most 2 minutes), restarts the LLM in
  the stage and tells the worker it is ready or paused
- the game stage runs at 16K context, one request at a time, half the engine's CPU threads and
  the game placement (reserve game.reserveGb, default 8, else 2 GiB, else 0 GiB, else pause),
  measured once at that context; every placement of a run counts other programs' memory as it
  was when the services started
- the voice worker gates every LLM use: on switching new work waits (at most 180 s), the first
  spoken request that waits hears the notice once under its own id, on ready all go on, on
  paused nothing is answered
- dashboard: "성능 단계: 게임 중 · 게임에 X GB" / 일시정지; game mode screen: game VRAM, revert
  wait, pause in game

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-14 10:23:21 +09:00

435 lines
21 KiB
Rust

//! The worker client against a stand-in worker server (hello, events, reconnect, failed TTS).
use std::sync::{Arc, Mutex};
use std::time::Duration;
use futures_util::{SinkExt, StreamExt};
use jarvis_app::frames::{encode_audio_frame, KIND_TTS};
use jarvis_app::playback::TtsBackend;
use jarvis_app::settings::ChatChannelSettings;
use jarvis_app::worker_client::{ChatAction, TtsDone, WorkerClient, WorkerClientOptions, WorkerEvent};
use serde_json::{json, Value};
use tokio::net::TcpListener;
use tokio::sync::mpsc;
use tokio_tungstenite::tungstenite::Message;
const TOKEN: &str = "t0ken";
#[derive(Debug, Clone, PartialEq)]
enum Got {
Json(Value),
Bin(Vec<u8>),
}
/// A fake worker: checks the hello, records what arrives, and lets the test push frames.
struct FakeWorker {
port: u16,
received: Arc<Mutex<Vec<Got>>>,
push: Arc<Mutex<Option<mpsc::UnboundedSender<Message>>>>,
task: tokio::task::JoinHandle<()>,
conns: Arc<Mutex<Vec<tokio::task::JoinHandle<()>>>>,
}
impl FakeWorker {
async fn start(port: u16) -> Self {
let listener = TcpListener::bind(("127.0.0.1", port)).await.unwrap();
let port = listener.local_addr().unwrap().port();
let received = Arc::new(Mutex::new(vec![]));
let push = Arc::new(Mutex::new(None));
let conns = Arc::new(Mutex::new(vec![]));
let (r, p, cs) = (received.clone(), push.clone(), conns.clone());
let task = tokio::spawn(async move {
while let Ok((tcp, _)) = listener.accept().await {
let ws = tokio_tungstenite::accept_async(tcp).await.unwrap();
let (mut tx, mut rx) = ws.split();
let (ptx, mut prx) = mpsc::unbounded_channel::<Message>();
*p.lock().unwrap() = Some(ptx.clone());
let r = r.clone();
cs.lock().unwrap().push(tokio::spawn(async move {
loop {
tokio::select! {
m = prx.recv() => match m { Some(m) => { if tx.send(m).await.is_err() { break } } None => break },
m = rx.next() => match m {
Some(Ok(Message::Text(t))) => {
let v: Value = serde_json::from_str(t.as_str()).unwrap();
if v["type"] == "hello" {
if v["token"] != TOKEN { let _ = tx.close().await; break; }
let ok = json!({"type": "hello_ok", "stt_ready": true, "tts_ready": true, "stt": v["stt"]});
let _ = ptx.send(Message::text(ok.to_string()));
}
r.lock().unwrap().push(Got::Json(v));
}
Some(Ok(Message::Binary(b))) => r.lock().unwrap().push(Got::Bin(b.to_vec())),
Some(Ok(_)) => {}
_ => break,
},
}
}
}));
}
});
Self { port, received, push, task, conns }
}
fn url(&self) -> String {
format!("ws://127.0.0.1:{}", self.port)
}
fn got(&self) -> Vec<Got> {
self.received.lock().unwrap().clone()
}
fn send(&self, m: Message) {
self.push.lock().unwrap().as_ref().unwrap().send(m).unwrap();
}
/// The worker dies: stop listening and drop the connection.
fn kill(self) -> u16 {
self.task.abort();
for c in self.conns.lock().unwrap().drain(..) {
c.abort();
}
*self.push.lock().unwrap() = None;
self.port
}
}
fn client(url: &str, stt: Value) -> (WorkerClient, mpsc::UnboundedReceiver<WorkerEvent>) {
let mut o = WorkerClientOptions::new(url, TOKEN);
o.stt = stt.as_object().cloned().unwrap_or_default();
o.backoff = Duration::from_millis(20);
o.max_backoff = Duration::from_millis(50);
WorkerClient::new(o)
}
async fn until(mut pred: impl FnMut() -> bool) {
tokio::time::timeout(Duration::from_secs(3), async {
while !pred() {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("timed out");
}
async fn next_matching(rx: &mut mpsc::UnboundedReceiver<WorkerEvent>, f: impl Fn(&WorkerEvent) -> bool) -> WorkerEvent {
tokio::time::timeout(Duration::from_secs(3), async {
loop {
let e = rx.recv().await.expect("events closed");
if f(&e) {
return e;
}
}
})
.await
.expect("timed out")
}
fn tts_frame(id: &str, seq: u32) -> Message {
let mut f = vec![KIND_TTS, id.len() as u8];
f.extend_from_slice(id.as_bytes());
f.extend_from_slice(&seq.to_le_bytes());
f.extend_from_slice(&[0; 8]);
Message::binary(f)
}
#[tokio::test]
async fn hello_carries_token_formats_and_stt_then_audio_flows() {
let w = FakeWorker::start(0).await;
let (c, _rx) = client(&w.url(), json!({"hotwords": ["자비스"]}));
c.start();
let ok = c.wait_ready(Duration::from_secs(2)).await.expect("ready");
assert_eq!(Value::Object(ok.stt), json!({"hotwords": ["자비스"]}));
let pcm = vec![1i16; 1920];
c.send_audio("123", &pcm);
c.speaking_end("123");
c.speaker_leave("123");
until(|| w.got().len() >= 4).await;
let got = w.got();
assert_eq!(got[0], Got::Json(json!({"type": "hello", "token": TOKEN, "audio": {"rate": 48000, "channels": 2},
"tts": {"rate": 48000, "channels": 2}, "stt": {"hotwords": ["자비스"]}})));
assert_eq!(got[1], Got::Bin(encode_audio_frame("123", &pcm).unwrap()));
assert_eq!(got[2..], [Got::Json(json!({"type": "speaking_end", "speaker": "123"})),
Got::Json(json!({"type": "speaker_leave", "speaker": "123"}))]);
c.close();
}
#[tokio::test]
async fn hello_carries_the_memory_settings_when_there_are_any() {
let w = FakeWorker::start(0).await;
let mut o = WorkerClientOptions::new(w.url(), TOKEN);
o.memory = json!({"others": false, "recent_records": 5}).as_object().cloned().unwrap();
let (c, _rx) = WorkerClient::new(o);
c.start();
c.wait_ready(Duration::from_secs(2)).await.expect("ready");
let Got::Json(hello) = &w.got()[0] else { panic!("hello first") };
assert_eq!(hello["memory"], json!({"others": false, "recent_records": 5}));
c.close();
}
#[tokio::test]
async fn events_pass_on_and_only_tts_audio_of_open_requests() {
let w = FakeWorker::start(0).await;
let (c, mut rx) = client(&w.url(), json!({}));
c.start();
c.wait_ready(Duration::from_secs(2)).await.unwrap();
assert!(c.tts("r1", "안녕하세요.", false));
until(|| w.got().iter().any(|g| matches!(g, Got::Json(v) if v["type"] == "tts"))).await;
w.send(tts_frame("zz", 0)); // not ours
w.send(tts_frame("r1", 0));
w.send(Message::text(json!({"type": "final", "speaker": "1", "utt": 1, "text": "네", "end_reason": "silence",
"audio_ms": 900, "wait_ms": 100, "stt_ms": 80}).to_string()));
w.send(Message::text(json!({"type": "tts_done", "id": "r1", "chunks": 1, "cancelled": false, "failed": false}).to_string()));
let mut seen = vec![];
while seen.len() < 4 {
let e = next_matching(&mut rx, |_| true).await;
if !matches!(e, WorkerEvent::Ready(_)) {
seen.push(e);
}
if matches!(seen.last(), Some(WorkerEvent::TtsDone(_))) {
break;
}
}
assert!(matches!(&seen[0], WorkerEvent::TtsAudio { id, seq: 0, .. } if id == "r1"));
assert!(matches!(&seen[1], WorkerEvent::Final(f) if f.text == "네" && f.end_reason == "silence"));
assert_eq!(seen[2], WorkerEvent::TtsDone(TtsDone { id: "r1".into(), chunks: 1, cancelled: false, failed: false }));
c.close();
}
#[tokio::test]
async fn a_refused_tts_request_ends_as_failed() {
let w = FakeWorker::start(0).await;
let (c, mut rx) = client(&w.url(), json!({}));
c.start();
c.wait_ready(Duration::from_secs(2)).await.unwrap();
c.tts("r9", "x", false);
until(|| w.got().len() >= 2).await;
w.send(Message::text(json!({"type": "error", "code": "tts_unavailable", "message": "no", "id": "r9"}).to_string()));
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::TtsDone(_))).await;
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r9".into(), chunks: 0, cancelled: false, failed: true }));
c.close();
}
#[tokio::test]
async fn a_worker_error_reaches_the_log_hub_without_debug_logging() {
assert_ne!(std::env::var("JARVIS_DEBUG").as_deref(), Ok("1"), "run without debug logging");
let hub = jarvis_app::logs::LogHub::new(None);
jarvis_app::logs::install(hub.clone());
let hub = jarvis_app::logs::installed().unwrap();
let w = FakeWorker::start(0).await;
let (c, mut rx) = client(&w.url(), json!({}));
c.start();
c.wait_ready(Duration::from_secs(2)).await.unwrap();
w.send(Message::text(json!({"type": "error", "code": "tts_failed", "message": "engine broke", "id": "r7"}).to_string()));
next_matching(&mut rx, |e| matches!(e, WorkerEvent::Error(_))).await;
let errors = hub.query(Some("worker"), jarvis_app::logs::Level::Error, Some("tts_failed"), None, 10);
assert_eq!(errors.len(), 1, "{errors:?}");
assert!(errors[0].text.contains("r7"));
assert!(hub.recent_error_report("test").unwrap().contains("tts_failed"));
c.close();
}
#[tokio::test]
async fn reconnects_after_a_restart_and_fails_the_tts_in_flight() {
let w = FakeWorker::start(0).await;
let (c, mut rx) = client(&w.url(), json!({"language": "ko"}));
c.start();
c.wait_ready(Duration::from_secs(2)).await.unwrap();
c.tts("r1", "길게 말하는 중", false);
c.configure(json!({"partials": false}).as_object().unwrap().clone());
until(|| w.got().len() >= 3).await;
assert_eq!(w.got()[2], Got::Json(json!({"type": "config", "partials": false})));
let port = w.kill();
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::TtsDone(_))).await;
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r1".into(), chunks: 0, cancelled: false, failed: true }));
next_matching(&mut rx, |e| *e == WorkerEvent::Disconnected).await;
assert!(!c.is_ready());
c.send_audio("1", &[0, 0]); // dropped, not queued
let w = FakeWorker::start(port).await;
next_matching(&mut rx, |e| matches!(e, WorkerEvent::Ready(_))).await;
let got = w.got();
// The new hello carries the settings changed since, and no stale audio was queued.
assert!(matches!(&got[0], Got::Json(v) if v["stt"] == json!({"language": "ko", "partials": false})));
assert!(!got.iter().any(|g| matches!(g, Got::Bin(_))));
c.close();
}
#[tokio::test]
async fn tts_while_disconnected_fails_at_once() {
let (c, mut rx) = client("ws://127.0.0.1:1", json!({}));
assert!(!c.tts("r1", "x", false));
let e = next_matching(&mut rx, |_| true).await;
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r1".into(), chunks: 0, cancelled: false, failed: true }));
}
#[tokio::test]
async fn history_messages_follow_the_worker_protocol() {
let w = FakeWorker::start(0).await;
let (c, _rx) = client(&w.url(), json!({}));
c.start();
c.wait_ready(Duration::from_secs(2)).await.expect("ready");
c.voice_join(42);
c.member_join(7, "Alice");
c.member_leave(7);
c.chat_message(9, 100, 7, "Alice", "hi there", 1_800_000_000.5, None);
c.chat_message(9, 101, 1, "Jarvis", "hello", 1_800_000_001.0, Some(7));
c.chat_edit(9, 100, "hi, edited");
c.chat_delete(9, 100);
c.voice_leave();
until(|| w.got().len() >= 9).await;
let got: Vec<Value> = w.got()[1..].iter().map(|g| match g { Got::Json(v) => v.clone(), Got::Bin(_) => json!(null) }).collect();
assert_eq!(got, vec![
json!({"type": "voice_join", "channel": "42"}),
json!({"type": "member_join", "user": "7", "name": "Alice"}),
json!({"type": "member_leave", "user": "7"}),
json!({"type": "chat_message", "channel": "9", "message_id": "100", "author": "7", "name": "Alice",
"text": "hi there", "ts": 1_800_000_000.5}),
json!({"type": "chat_message", "channel": "9", "message_id": "101", "author": "1", "name": "Jarvis",
"text": "hello", "ts": 1_800_000_001.0, "to": "7"}),
json!({"type": "chat_edit", "channel": "9", "message_id": "100", "text": "hi, edited"}),
json!({"type": "chat_delete", "channel": "9", "message_id": "100"}),
json!({"type": "voice_leave"}),
]);
c.close();
}
#[tokio::test]
async fn chat_channels_go_out_and_chat_actions_come_back_with_their_results() {
let w = FakeWorker::start(0).await;
let (c, mut rx) = client(&w.url(), json!({}));
c.start();
c.wait_ready(Duration::from_secs(2)).await.expect("ready");
c.chat_channels(Some(900), &[ChatChannelSettings { id: 5, name: "일정".into(), description: "약속".into(),
read: true, send: true, edit: true, react: true, delete: false, pin: false, answer_all: false, sync_discord: true }]);
w.send(Message::text(json!({"type": "chat_action", "id": "a1", "action": "react", "channel": "5",
"message_id": "77", "emoji": "👍"}).to_string()));
let ev = next_matching(&mut rx, |e| matches!(e, WorkerEvent::ChatAction(_))).await;
assert_eq!(ev, WorkerEvent::ChatAction(ChatAction { id: "a1".into(), action: "react".into(), channel: "5".into(),
message_id: Some("77".into()), text: None, emoji: Some("👍".into()) }));
c.chat_action_result("a1", Ok(vec![77]));
c.chat_action_result("a2", Err("send is turned off".into()));
until(|| w.got().len() >= 4).await;
let got: Vec<Value> = w.got()[1..].iter().map(|g| match g { Got::Json(v) => v.clone(), Got::Bin(_) => json!(null) }).collect();
assert_eq!(got, vec![
json!({"type": "chat_channels", "bot": "900", "channels": [{"id": "5", "name": "일정", "description": "약속",
"read": true, "send": true, "edit": true, "react": true, "delete": false, "pin": false}]}),
json!({"type": "chat_action_result", "id": "a1", "ok": true, "message_ids": ["77"]}),
json!({"type": "chat_action_result", "id": "a2", "ok": false, "error": "send is turned off"}),
]);
c.close();
}
#[tokio::test]
async fn a_spoken_reply_takes_its_audio_and_a_failed_one_ends_its_speech() {
let w = FakeWorker::start(0).await;
let (c, mut rx) = client(&w.url(), json!({}));
c.start();
c.wait_ready(Duration::from_secs(2)).await.expect("ready");
assert!(c.reply("r1", 42, "자비스 날씨 어때", "voice", true));
assert!(c.reply("r2", 43, "자비스 안녕", "voice", true));
until(|| w.got().len() >= 3).await;
assert_eq!(w.got()[1], Got::Json(json!({"type": "reply", "id": "r1", "user": "42", "text": "자비스 날씨 어때",
"via": "voice", "speak": true})));
// The brain's answer is spoken under the reply id, so its audio reaches the app.
w.send(Message::text(json!({"type": "reply_text", "id": "r1", "text": "맑아요.", "failed": false}).to_string()));
w.send(tts_frame("r1", 0));
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::ReplyText { .. })).await;
assert_eq!(e, WorkerEvent::ReplyText { id: "r1".into(), text: "맑아요.".into(), failed: false });
next_matching(&mut rx, |e| matches!(e, WorkerEvent::TtsAudio { id, .. } if id == "r1")).await;
// No answer: nothing will be spoken, so its speech ends as failed.
w.send(Message::text(json!({"type": "reply_text", "id": "r2", "text": "", "failed": true}).to_string()));
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::TtsDone(_))).await;
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r2".into(), chunks: 0, cancelled: false, failed: true }));
c.close();
}
#[tokio::test]
async fn a_reply_the_worker_cannot_answer_or_send_ends_as_failed() {
let w = FakeWorker::start(0).await;
let (c, mut rx) = client(&w.url(), json!({}));
c.start();
c.wait_ready(Duration::from_secs(2)).await.expect("ready");
c.reply("r3", 42, "x", "voice", true);
w.send(Message::text(json!({"type": "error", "code": "brain_unavailable", "message": "no", "id": "r3"}).to_string()));
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::TtsDone(_))).await;
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r3".into(), chunks: 0, cancelled: false, failed: true }));
c.close();
let (c, mut rx) = client("ws://127.0.0.1:1", json!({}));
assert!(!c.reply("r4", 42, "x", "voice", true));
let e = next_matching(&mut rx, |_| true).await;
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r4".into(), chunks: 0, cancelled: false, failed: true }));
}
#[tokio::test]
async fn addressed_and_rank_questions_go_out_and_their_answers_come_back() {
let w = FakeWorker::start(0).await;
let (c, mut rx) = client(&w.url(), json!({}));
c.start();
c.wait_ready(Duration::from_secs(2)).await.expect("ready");
assert!(c.addressed("q1", 42, "불 좀 꺼 줄래"));
assert!(c.rank("k2", &[("7".into(), 42, "날씨".into()), ("8".into(), 43, "불 꺼".into())]));
until(|| w.got().len() >= 3).await;
let got: Vec<Value> = w.got()[1..].iter().map(|g| match g { Got::Json(v) => v.clone(), Got::Bin(_) => json!(null) }).collect();
assert_eq!(got, vec![
json!({"type": "addressed", "id": "q1", "user": "42", "text": "불 좀 꺼 줄래"}),
json!({"type": "rank", "id": "k2", "candidates": [{"id": "7", "user": "42", "text": "날씨"},
{"id": "8", "user": "43", "text": "불 꺼"}]}),
]);
w.send(Message::text(json!({"type": "addressed_result", "id": "q1", "addressed": true, "failed": false}).to_string()));
w.send(Message::text(json!({"type": "rank_result", "id": "k2", "order": ["8", "7"], "failed": false}).to_string()));
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::Addressed { .. })).await;
assert_eq!(e, WorkerEvent::Addressed { id: "q1".into(), addressed: true });
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::Ranked { .. })).await;
assert_eq!(e, WorkerEvent::Ranked { id: "k2".into(), order: vec!["8".into(), "7".into()] });
// A failed judgement is a "no", and a failed ranking has no order.
w.send(Message::text(json!({"type": "addressed_result", "id": "q3", "addressed": true, "failed": true}).to_string()));
w.send(Message::text(json!({"type": "rank_result", "id": "k4", "order": ["8"], "failed": true}).to_string()));
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::Addressed { .. })).await;
assert_eq!(e, WorkerEvent::Addressed { id: "q3".into(), addressed: false });
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::Ranked { .. })).await;
assert_eq!(e, WorkerEvent::Ranked { id: "k4".into(), order: vec![] });
c.close();
}
#[tokio::test]
async fn a_chat_reply_is_not_spoken_and_its_text_comes_back() {
let w = FakeWorker::start(0).await;
let (c, mut rx) = client(&w.url(), json!({}));
c.start();
c.wait_ready(Duration::from_secs(2)).await.expect("ready");
assert!(c.reply("c1", 42, "자비스 일정", "chat", false));
until(|| w.got().len() >= 2).await;
assert_eq!(w.got()[1], Got::Json(json!({"type": "reply", "id": "c1", "user": "42", "text": "자비스 일정",
"via": "chat", "speak": false})));
w.send(tts_frame("c1", 0)); // not a speech the app asked for
w.send(Message::text(json!({"type": "reply_text", "id": "c1", "text": "", "failed": true}).to_string()));
let e = next_matching(&mut rx, |e| !matches!(e, WorkerEvent::Ready(_))).await;
assert_eq!(e, WorkerEvent::ReplyText { id: "c1".into(), text: String::new(), failed: true });
// Whether the answer was posted goes back, so a question in it counts only once it was seen.
c.reply_delivered("c1", true);
c.reply_delivered("c2", false);
until(|| w.got().len() >= 4).await;
assert_eq!(w.got()[2..], [Got::Json(json!({"type": "reply_delivered", "id": "c1", "ok": true})),
Got::Json(json!({"type": "reply_delivered", "id": "c2", "ok": false}))]);
c.close();
}
#[tokio::test]
async fn an_llm_switch_is_announced_and_the_client_waits_until_the_brain_is_idle() {
let w = FakeWorker::start(0).await;
let (c, _rx) = client(&w.url(), json!({}));
c.start();
c.wait_ready(Duration::from_secs(2)).await.unwrap();
assert!(c.llm_state("switching", Some("잠깐만요")));
until(|| w.got().iter().any(|g| *g == Got::Json(json!({"type": "llm", "state": "switching", "notice": "잠깐만요"})))).await;
// Nothing idle yet: the wait gives up after its time.
assert!(!c.wait_llm_idle(Duration::from_millis(100)).await);
// The worker reports idle (before or while the app waits): the wait ends.
w.send(Message::text(json!({"type": "llm_idle"}).to_string()));
assert!(c.wait_llm_idle(Duration::from_secs(2)).await);
assert!(c.llm_state("ready", None));
until(|| w.got().iter().any(|g| *g == Got::Json(json!({"type": "llm", "state": "ready"})))).await;
c.close();
}