The user asked for a faster, real app instead of Electron. The app is now a Rust crate (serenity + songbird); the GUI in M5 will be a Tauri 2 shell on the system WebView2. - Same behaviour as the Electron core: one voice channel, per-speaker PCM to the voice worker, speaking_end and speaker_leave, sentence playback with stop and cancel, transcription notice on join, slash command join/leave/stop (create-or-update), registered chat channels with live events, history and sends that never ping, worker process with a fresh token and restart backoff. - songbird 0.6.1 garbles about a third of received frames under DAVE (its decoder keeps the bytes DAVE decryption leaves at the end of the payload; a 20 ms stepped test tone showed repeats, skips and reordering, while sending was exact). songbird now only decrypts; the core reorders packets in a small window (Opus FEC for the last lost frame) and decodes them with libopus itself. - Live test on real Discord with two bots: five sentences heard (CER 0/0/0/5/12.5%), 0.71-0.90 s from end of speech to final, speech read back at 3.4%, sound gone 0.08 s after stop. The release binary uses about 16 MB of memory; a Windows build (MinGW) logs in under Wine. - Spec moved to app/src/discord_core.spec.md; architecture, decisions, roadmap and pc-features updated. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
231 lines
9.3 KiB
Rust
231 lines
9.3 KiB
Rust
//! The worker client against a stand-in worker server (hello, events, reconnect, failed TTS).
|
|
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::Duration;
|
|
|
|
use futures_util::{SinkExt, StreamExt};
|
|
use jarvis_app::frames::{encode_audio_frame, KIND_TTS};
|
|
use jarvis_app::playback::TtsBackend;
|
|
use jarvis_app::worker_client::{TtsDone, WorkerClient, WorkerClientOptions, WorkerEvent};
|
|
use serde_json::{json, Value};
|
|
use tokio::net::TcpListener;
|
|
use tokio::sync::mpsc;
|
|
use tokio_tungstenite::tungstenite::Message;
|
|
|
|
const TOKEN: &str = "t0ken";
|
|
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
enum Got {
|
|
Json(Value),
|
|
Bin(Vec<u8>),
|
|
}
|
|
|
|
/// A fake worker: checks the hello, records what arrives, and lets the test push frames.
|
|
struct FakeWorker {
|
|
port: u16,
|
|
received: Arc<Mutex<Vec<Got>>>,
|
|
push: Arc<Mutex<Option<mpsc::UnboundedSender<Message>>>>,
|
|
task: tokio::task::JoinHandle<()>,
|
|
conns: Arc<Mutex<Vec<tokio::task::JoinHandle<()>>>>,
|
|
}
|
|
|
|
impl FakeWorker {
|
|
async fn start(port: u16) -> Self {
|
|
let listener = TcpListener::bind(("127.0.0.1", port)).await.unwrap();
|
|
let port = listener.local_addr().unwrap().port();
|
|
let received = Arc::new(Mutex::new(vec![]));
|
|
let push = Arc::new(Mutex::new(None));
|
|
let conns = Arc::new(Mutex::new(vec![]));
|
|
let (r, p, cs) = (received.clone(), push.clone(), conns.clone());
|
|
let task = tokio::spawn(async move {
|
|
while let Ok((tcp, _)) = listener.accept().await {
|
|
let ws = tokio_tungstenite::accept_async(tcp).await.unwrap();
|
|
let (mut tx, mut rx) = ws.split();
|
|
let (ptx, mut prx) = mpsc::unbounded_channel::<Message>();
|
|
*p.lock().unwrap() = Some(ptx.clone());
|
|
let r = r.clone();
|
|
cs.lock().unwrap().push(tokio::spawn(async move {
|
|
loop {
|
|
tokio::select! {
|
|
m = prx.recv() => match m { Some(m) => { if tx.send(m).await.is_err() { break } } None => break },
|
|
m = rx.next() => match m {
|
|
Some(Ok(Message::Text(t))) => {
|
|
let v: Value = serde_json::from_str(t.as_str()).unwrap();
|
|
if v["type"] == "hello" {
|
|
if v["token"] != TOKEN { let _ = tx.close().await; break; }
|
|
let ok = json!({"type": "hello_ok", "stt_ready": true, "tts_ready": true, "stt": v["stt"]});
|
|
let _ = ptx.send(Message::text(ok.to_string()));
|
|
}
|
|
r.lock().unwrap().push(Got::Json(v));
|
|
}
|
|
Some(Ok(Message::Binary(b))) => r.lock().unwrap().push(Got::Bin(b.to_vec())),
|
|
Some(Ok(_)) => {}
|
|
_ => break,
|
|
},
|
|
}
|
|
}
|
|
}));
|
|
}
|
|
});
|
|
Self { port, received, push, task, conns }
|
|
}
|
|
|
|
fn url(&self) -> String {
|
|
format!("ws://127.0.0.1:{}", self.port)
|
|
}
|
|
|
|
fn got(&self) -> Vec<Got> {
|
|
self.received.lock().unwrap().clone()
|
|
}
|
|
|
|
fn send(&self, m: Message) {
|
|
self.push.lock().unwrap().as_ref().unwrap().send(m).unwrap();
|
|
}
|
|
|
|
/// The worker dies: stop listening and drop the connection.
|
|
fn kill(self) -> u16 {
|
|
self.task.abort();
|
|
for c in self.conns.lock().unwrap().drain(..) {
|
|
c.abort();
|
|
}
|
|
*self.push.lock().unwrap() = None;
|
|
self.port
|
|
}
|
|
}
|
|
|
|
fn client(url: &str, stt: Value) -> (WorkerClient, mpsc::UnboundedReceiver<WorkerEvent>) {
|
|
let mut o = WorkerClientOptions::new(url, TOKEN);
|
|
o.stt = stt.as_object().cloned().unwrap_or_default();
|
|
o.backoff = Duration::from_millis(20);
|
|
o.max_backoff = Duration::from_millis(50);
|
|
WorkerClient::new(o)
|
|
}
|
|
|
|
async fn until(mut pred: impl FnMut() -> bool) {
|
|
tokio::time::timeout(Duration::from_secs(3), async {
|
|
while !pred() {
|
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
|
}
|
|
})
|
|
.await
|
|
.expect("timed out");
|
|
}
|
|
|
|
async fn next_matching(rx: &mut mpsc::UnboundedReceiver<WorkerEvent>, f: impl Fn(&WorkerEvent) -> bool) -> WorkerEvent {
|
|
tokio::time::timeout(Duration::from_secs(3), async {
|
|
loop {
|
|
let e = rx.recv().await.expect("events closed");
|
|
if f(&e) {
|
|
return e;
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.expect("timed out")
|
|
}
|
|
|
|
fn tts_frame(id: &str, seq: u32) -> Message {
|
|
let mut f = vec![KIND_TTS, id.len() as u8];
|
|
f.extend_from_slice(id.as_bytes());
|
|
f.extend_from_slice(&seq.to_le_bytes());
|
|
f.extend_from_slice(&[0; 8]);
|
|
Message::binary(f)
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn hello_carries_token_formats_and_stt_then_audio_flows() {
|
|
let w = FakeWorker::start(0).await;
|
|
let (c, _rx) = client(&w.url(), json!({"hotwords": ["자비스"]}));
|
|
c.start();
|
|
let ok = c.wait_ready(Duration::from_secs(2)).await.expect("ready");
|
|
assert_eq!(Value::Object(ok.stt), json!({"hotwords": ["자비스"]}));
|
|
let pcm = vec![1i16; 1920];
|
|
c.send_audio("123", &pcm);
|
|
c.speaking_end("123");
|
|
c.speaker_leave("123");
|
|
until(|| w.got().len() >= 4).await;
|
|
let got = w.got();
|
|
assert_eq!(got[0], Got::Json(json!({"type": "hello", "token": TOKEN, "audio": {"rate": 48000, "channels": 2},
|
|
"tts": {"rate": 48000, "channels": 2}, "stt": {"hotwords": ["자비스"]}})));
|
|
assert_eq!(got[1], Got::Bin(encode_audio_frame("123", &pcm).unwrap()));
|
|
assert_eq!(got[2..], [Got::Json(json!({"type": "speaking_end", "speaker": "123"})),
|
|
Got::Json(json!({"type": "speaker_leave", "speaker": "123"}))]);
|
|
c.close();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn events_pass_on_and_only_tts_audio_of_open_requests() {
|
|
let w = FakeWorker::start(0).await;
|
|
let (c, mut rx) = client(&w.url(), json!({}));
|
|
c.start();
|
|
c.wait_ready(Duration::from_secs(2)).await.unwrap();
|
|
assert!(c.tts("r1", "안녕하세요.", false));
|
|
until(|| w.got().iter().any(|g| matches!(g, Got::Json(v) if v["type"] == "tts"))).await;
|
|
w.send(tts_frame("zz", 0)); // not ours
|
|
w.send(tts_frame("r1", 0));
|
|
w.send(Message::text(json!({"type": "final", "speaker": "1", "utt": 1, "text": "네", "end_reason": "silence",
|
|
"audio_ms": 900, "wait_ms": 100, "stt_ms": 80}).to_string()));
|
|
w.send(Message::text(json!({"type": "tts_done", "id": "r1", "chunks": 1, "cancelled": false, "failed": false}).to_string()));
|
|
let mut seen = vec![];
|
|
while seen.len() < 4 {
|
|
let e = next_matching(&mut rx, |_| true).await;
|
|
if !matches!(e, WorkerEvent::Ready(_)) {
|
|
seen.push(e);
|
|
}
|
|
if matches!(seen.last(), Some(WorkerEvent::TtsDone(_))) {
|
|
break;
|
|
}
|
|
}
|
|
assert!(matches!(&seen[0], WorkerEvent::TtsAudio { id, seq: 0, .. } if id == "r1"));
|
|
assert!(matches!(&seen[1], WorkerEvent::Final(f) if f.text == "네" && f.end_reason == "silence"));
|
|
assert_eq!(seen[2], WorkerEvent::TtsDone(TtsDone { id: "r1".into(), chunks: 1, cancelled: false, failed: false }));
|
|
c.close();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_refused_tts_request_ends_as_failed() {
|
|
let w = FakeWorker::start(0).await;
|
|
let (c, mut rx) = client(&w.url(), json!({}));
|
|
c.start();
|
|
c.wait_ready(Duration::from_secs(2)).await.unwrap();
|
|
c.tts("r9", "x", false);
|
|
until(|| w.got().len() >= 2).await;
|
|
w.send(Message::text(json!({"type": "error", "code": "tts_unavailable", "message": "no", "id": "r9"}).to_string()));
|
|
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::TtsDone(_))).await;
|
|
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r9".into(), chunks: 0, cancelled: false, failed: true }));
|
|
c.close();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn reconnects_after_a_restart_and_fails_the_tts_in_flight() {
|
|
let w = FakeWorker::start(0).await;
|
|
let (c, mut rx) = client(&w.url(), json!({"language": "ko"}));
|
|
c.start();
|
|
c.wait_ready(Duration::from_secs(2)).await.unwrap();
|
|
c.tts("r1", "길게 말하는 중", false);
|
|
c.configure(json!({"partials": false}).as_object().unwrap().clone());
|
|
until(|| w.got().len() >= 3).await;
|
|
assert_eq!(w.got()[2], Got::Json(json!({"type": "config", "partials": false})));
|
|
let port = w.kill();
|
|
let e = next_matching(&mut rx, |e| matches!(e, WorkerEvent::TtsDone(_))).await;
|
|
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r1".into(), chunks: 0, cancelled: false, failed: true }));
|
|
next_matching(&mut rx, |e| *e == WorkerEvent::Disconnected).await;
|
|
assert!(!c.is_ready());
|
|
c.send_audio("1", &[0, 0]); // dropped, not queued
|
|
let w = FakeWorker::start(port).await;
|
|
next_matching(&mut rx, |e| matches!(e, WorkerEvent::Ready(_))).await;
|
|
let got = w.got();
|
|
// The new hello carries the settings changed since, and no stale audio was queued.
|
|
assert!(matches!(&got[0], Got::Json(v) if v["stt"] == json!({"language": "ko", "partials": false})));
|
|
assert!(!got.iter().any(|g| matches!(g, Got::Bin(_))));
|
|
c.close();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn tts_while_disconnected_fails_at_once() {
|
|
let (c, mut rx) = client("ws://127.0.0.1:1", json!({}));
|
|
assert!(!c.tts("r1", "x", false));
|
|
let e = next_matching(&mut rx, |_| true).await;
|
|
assert_eq!(e, WorkerEvent::TtsDone(TtsDone { id: "r1".into(), chunks: 0, cancelled: false, failed: true }));
|
|
}
|