The user asked for a faster, real app instead of Electron. The app is now a Rust crate (serenity + songbird); the GUI in M5 will be a Tauri 2 shell on the system WebView2. - Same behaviour as the Electron core: one voice channel, per-speaker PCM to the voice worker, speaking_end and speaker_leave, sentence playback with stop and cancel, transcription notice on join, slash command join/leave/stop (create-or-update), registered chat channels with live events, history and sends that never ping, worker process with a fresh token and restart backoff. - songbird 0.6.1 garbles about a third of received frames under DAVE (its decoder keeps the bytes DAVE decryption leaves at the end of the payload; a 20 ms stepped test tone showed repeats, skips and reordering, while sending was exact). songbird now only decrypts; the core reorders packets in a small window (Opus FEC for the last lost frame) and decodes them with libopus itself. - Live test on real Discord with two bots: five sentences heard (CER 0/0/0/5/12.5%), 0.71-0.90 s from end of speech to final, speech read back at 3.4%, sound gone 0.08 s after stop. The release binary uses about 16 MB of memory; a Windows build (MinGW) logs in under Wine. - Spec moved to app/src/discord_core.spec.md; architecture, decisions, roadmap and pc-features updated. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
111 lines
4.1 KiB
Rust
111 lines
4.1 KiB
Rust
//! The worker client and playback against the real Python voice worker server with fake engines, so
|
|
//! both sides of the protocol are checked together. Needs a Python with the worker's requirements:
|
|
//! JARVIS_VOICE_PYTHON=<venv python> cargo test --test worker_protocol
|
|
|
|
use std::path::Path;
|
|
use std::time::Duration;
|
|
|
|
use jarvis_app::playback::{ClipSink, Playback, PlaybackEvent};
|
|
use jarvis_app::worker_client::{WorkerClient, WorkerClientOptions, WorkerEvent};
|
|
use jarvis_app::worker_process::free_port;
|
|
use serde_json::json;
|
|
use tokio::io::{AsyncBufReadExt, BufReader};
|
|
|
|
const TOKEN: &str = "protocol-test-token";
|
|
|
|
/// 20 ms of a 48 kHz stereo tone whose amplitude and pitch (x1000 Hz) are `level`.
|
|
fn frame(level: f64) -> Vec<i16> {
|
|
(0..960)
|
|
.flat_map(|i| {
|
|
let v = (level * (2.0 * std::f64::consts::PI * level * 1000.0 * i as f64 / 48000.0).sin() * 32767.0) as i16;
|
|
[v, v]
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
#[derive(Default)]
|
|
struct Sink(Vec<usize>);
|
|
impl ClipSink for Sink {
|
|
fn play(&mut self, pcm: Vec<u8>) {
|
|
self.0.push(pcm.len());
|
|
}
|
|
fn stop(&mut self) {}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn audio_and_tts_round_trip_through_the_python_server() {
|
|
let Ok(python) = std::env::var("JARVIS_VOICE_PYTHON") else {
|
|
eprintln!("skipped: set JARVIS_VOICE_PYTHON");
|
|
return;
|
|
};
|
|
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
|
let port = free_port().unwrap();
|
|
let mut child = tokio::process::Command::new(python)
|
|
.arg(root.join("tests/fixtures/fake_engine_worker.py"))
|
|
.arg(port.to_string())
|
|
.env("PYTHONPATH", root.join("../src"))
|
|
.env("JARVIS_VOICE_TOKEN", TOKEN)
|
|
.stdout(std::process::Stdio::piped())
|
|
.kill_on_drop(true)
|
|
.spawn()
|
|
.unwrap();
|
|
let mut lines = BufReader::new(child.stdout.take().unwrap()).lines();
|
|
while let Some(l) = lines.next_line().await.unwrap() {
|
|
if l.starts_with("READY") {
|
|
break;
|
|
}
|
|
}
|
|
let mut o = WorkerClientOptions::new(format!("ws://127.0.0.1:{port}"), TOKEN);
|
|
o.stt = json!({"partials": false, "hotwords": ["자비스"]}).as_object().unwrap().clone();
|
|
let (c, mut rx) = WorkerClient::new(o);
|
|
c.start();
|
|
let ok = c.wait_ready(Duration::from_secs(5)).await.expect("ready");
|
|
assert!(ok.stt_ready && ok.tts_ready);
|
|
assert_eq!(ok.stt["hotwords"], json!(["자비스"]));
|
|
|
|
// Two Discord-format speakers; the second leaves mid-utterance and still gets its final.
|
|
let (a, b) = ("111111111111111111", "222222222222222222");
|
|
for _ in 0..30 {
|
|
c.send_audio(a, &frame(0.9));
|
|
c.send_audio(b, &frame(0.6));
|
|
}
|
|
c.speaking_end(a);
|
|
c.speaker_leave(b);
|
|
let mut finals = std::collections::HashMap::new();
|
|
tokio::time::timeout(Duration::from_secs(5), async {
|
|
while finals.len() < 2 {
|
|
if let Some(WorkerEvent::Final(f)) = rx.recv().await {
|
|
finals.insert(f.speaker.clone(), f);
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.expect("finals");
|
|
assert_eq!((finals[a].text.as_str(), finals[a].end_reason.as_str(), finals[a].utt), ("level 0.9", "speaking_end", 1));
|
|
assert_eq!((finals[b].text.as_str(), finals[b].end_reason.as_str()), ("level 0.6", "speaking_end"));
|
|
|
|
// TTS sentences play in order through Playback; 10 samples per character, stereo, 2 bytes each.
|
|
let mut p = Playback::new(c.clone(), Sink::default());
|
|
p.say("첫 번째 문장입니다. 두 번째 문장은 조금 더 깁니다.", false);
|
|
let end = tokio::time::timeout(Duration::from_secs(5), async {
|
|
loop {
|
|
match rx.recv().await.unwrap() {
|
|
WorkerEvent::TtsAudio { id, seq, pcm } => {
|
|
p.audio(&id, seq, pcm);
|
|
p.clip_ended();
|
|
}
|
|
WorkerEvent::TtsDone(d) => p.done(&d),
|
|
_ => {}
|
|
}
|
|
if let Some(PlaybackEvent::End(e)) = p.take_events().into_iter().find(|e| matches!(e, PlaybackEvent::End(_))) {
|
|
return e;
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.expect("speech end");
|
|
assert_eq!((end.played, end.cancelled, end.failed), (2, false, false));
|
|
c.close();
|
|
let _ = child.kill().await;
|
|
}
|