test(app): time the whole conversation on real Discord, alone and with 9 more voices
live_round_trip.rs runs the app's services with the real worker and brain, plays recorded questions from a second bot and times the final, the answer text, playback start and the first audible frame after the speech ends; a second worker session streams nine more voices for the load run. Roadmap: first M4 step 4 numbers (median 3.5 s alone, 4.4 s under load). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
354
app/tests/live_round_trip.rs
Normal file
354
app/tests/live_round_trip.rs
Normal file
@@ -0,0 +1,354 @@
|
||||
//! M4 step 4: the whole conversation on real Discord, timed from the end of a question to Jarvis's
|
||||
//! first sound, with the real voice worker, the real brain (Strata) and the real response policy.
|
||||
//!
|
||||
//! The "speaker" bot plays recorded Korean questions that call Jarvis into the voice channel and
|
||||
//! listens for the answer. The Jarvis bot runs the app's services (`start_services`: core,
|
||||
//! conversation loop, worker process). Each question is timed at four points after the speech ends:
|
||||
//! the final transcript, the brain's answer text, playback start and the first audible frame the
|
||||
//! speaker bot receives. The rounds run again while nine more voices talk at the same time (a second
|
||||
//! worker session feeding the shared STT), which is the 10-person load.
|
||||
//! Opt-in; it refuses to run if a person is in the channel or either bot is already in voice.
|
||||
//!
|
||||
//! JARVIS_LIVE_ROUND_TRIP=1 JARVIS_LIVE_BOT_TOKEN=... JARVIS_LIVE_SPEAKER_TOKEN=... \
|
||||
//! JARVIS_LIVE_GUILD=<server id> JARVIS_LIVE_VOICE_CHANNEL=<voice channel id> \
|
||||
//! JARVIS_VOICE_PYTHON=<worker venv python> JARVIS_MELO_PYTHON=<MeloTTS venv python> \
|
||||
//! JARVIS_CONFIG_PATH=<jarvis settings whose ollama_base_url is Strata's /v1> \
|
||||
//! JARVIS_LIVE_QUESTIONS=<dir with refs.json and 16 kHz mono wavs> [JARVIS_LIVE_ROUNDS=2] \
|
||||
//! cargo test --release --test live_round_trip -- --nocapture
|
||||
//!
|
||||
//! On Linux, put the pip CUDA libraries on LD_LIBRARY_PATH (nvidia/cublas/lib, nvidia/cudnn/lib).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use jarvis_app::discord::CoreEvent;
|
||||
use jarvis_app::receive::{opus_payload, OpusFrames, SpeakerStream};
|
||||
use jarvis_app::services::start_services;
|
||||
use jarvis_app::worker_client::{WorkerClient, WorkerClientOptions};
|
||||
use serde_json::{json, Value};
|
||||
use serenity::all::{ChannelId, Client, Context, EventHandler, GatewayIntents, GuildId, Ready, UserId};
|
||||
use songbird::driver::DecodeMode;
|
||||
use songbird::input::Input;
|
||||
use songbird::{Config, CoreEvent as VoiceEv, Event, EventContext, EventHandler as VoiceHandler, SerenityInit, Songbird};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
fn env(k: &str) -> String {
|
||||
std::env::var(k).unwrap_or_else(|_| panic!("{k} is not set"))
|
||||
}
|
||||
|
||||
/// Seconds from the start of a 16 kHz mono PCM16 wav to its last sample above -34 dBFS.
|
||||
fn speech_end_seconds(wav: &[u8]) -> f64 {
|
||||
let data = &wav[44..];
|
||||
let n = data.len() / 2;
|
||||
(0..n).rev().find(|&k| i16::from_le_bytes([data[2 * k], data[2 * k + 1]]).unsigned_abs() > 655)
|
||||
.map_or(0.0, |k| (k + 1) as f64 / 16000.0)
|
||||
}
|
||||
|
||||
fn wav_samples(wav: &[u8]) -> Vec<i16> {
|
||||
wav[44..].chunks_exact(2).map(|b| i16::from_le_bytes([b[0], b[1]])).collect()
|
||||
}
|
||||
|
||||
struct HeardText {
|
||||
text: String,
|
||||
stt_ms: u64,
|
||||
}
|
||||
|
||||
struct ReadyOnce(mpsc::UnboundedSender<Ready>);
|
||||
#[async_trait]
|
||||
impl EventHandler for ReadyOnce {
|
||||
async fn ready(&self, _: Context, r: Ready) {
|
||||
let _ = self.0.send(r);
|
||||
}
|
||||
}
|
||||
|
||||
/// When Jarvis's voice first becomes audible to the speaker bot, decoded the way the core decodes
|
||||
/// (songbird's own decoder garbles DAVE audio, see src/receive.rs).
|
||||
#[derive(Clone, Default)]
|
||||
struct Ears {
|
||||
jarvis: u64,
|
||||
ssrc: Arc<Mutex<HashMap<u32, u64>>>,
|
||||
stream: Arc<Mutex<Option<SpeakerStream<OpusFrames>>>>,
|
||||
first_audible: Arc<Mutex<Option<Instant>>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl VoiceHandler for Ears {
|
||||
async fn act(&self, ctx: &EventContext<'_>) -> Option<Event> {
|
||||
match ctx {
|
||||
EventContext::SpeakingStateUpdate(s) => {
|
||||
if let Some(u) = s.user_id {
|
||||
self.ssrc.lock().unwrap().insert(s.ssrc, u.0);
|
||||
}
|
||||
}
|
||||
EventContext::RtpPacket(p) => {
|
||||
let (seq, ssrc, opus) = opus_payload(&p.packet, p.payload_offset, p.payload_end_pad)?;
|
||||
if self.ssrc.lock().unwrap().get(&ssrc) != Some(&self.jarvis) {
|
||||
return None;
|
||||
}
|
||||
let frames = self.stream.lock().unwrap().get_or_insert_with(|| SpeakerStream::new(OpusFrames::new()))
|
||||
.push(seq, opus.to_vec());
|
||||
for f in frames {
|
||||
// Audible: louder than -36 dBFS. Silence frames do not count.
|
||||
let rms = (f.iter().map(|&x| f64::from(x).powi(2)).sum::<f64>() / f.len().max(1) as f64).sqrt();
|
||||
if rms > 500.0 {
|
||||
self.first_audible.lock().unwrap().get_or_insert_with(Instant::now);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
async fn until(what: &str, ms: u64, mut pred: impl FnMut() -> bool) {
|
||||
let deadline = Instant::now() + Duration::from_millis(ms);
|
||||
while !pred() {
|
||||
assert!(Instant::now() < deadline, "timed out waiting for {what}");
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Nine more voices: a second worker session streams recorded sentences for nine speakers in real
|
||||
/// time, all talking at once with short pauses, until `stop` is set.
|
||||
fn background_voices(url: String, token: String, samples: Vec<Vec<i16>>, stop: Arc<AtomicBool>) -> tokio::task::JoinHandle<u64> {
|
||||
tokio::spawn(async move {
|
||||
let mut o = WorkerClientOptions::new(url, token);
|
||||
o.stt = json!({"partials": true}).as_object().unwrap().clone();
|
||||
let (load, mut rx) = WorkerClient::new(o);
|
||||
load.start();
|
||||
load.wait_ready(Duration::from_secs(10)).await.expect("load session hello");
|
||||
let finals = Arc::new(Mutex::new(0u64));
|
||||
let f2 = finals.clone();
|
||||
tokio::spawn(async move {
|
||||
while let Some(e) = rx.recv().await {
|
||||
if matches!(e, jarvis_app::worker_client::WorkerEvent::Final(_)) {
|
||||
*f2.lock().unwrap() += 1;
|
||||
}
|
||||
}
|
||||
});
|
||||
// 20 ms of 48 kHz stereo per frame, the format the core sends.
|
||||
let mut pos: Vec<(usize, usize, usize)> = (0..9).map(|i| (i % samples.len(), i * 4000, 0)).collect();
|
||||
let mut tick = tokio::time::interval(Duration::from_millis(20));
|
||||
while !stop.load(Ordering::Relaxed) {
|
||||
tick.tick().await;
|
||||
for (i, (sample, at, pause)) in pos.iter_mut().enumerate() {
|
||||
let speaker = format!("load-{i}");
|
||||
if *pause > 0 {
|
||||
*pause -= 1;
|
||||
if *pause == 0 {
|
||||
load.speaking_end(&speaker);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let s = &samples[*sample];
|
||||
let end = (*at + 320).min(s.len());
|
||||
let frame: Vec<i16> = s[*at..end].iter().flat_map(|&x| [x; 6]).collect();
|
||||
load.send_audio(&speaker, &frame);
|
||||
*at = end;
|
||||
if end == s.len() {
|
||||
*sample = (*sample + 1) % samples.len();
|
||||
*at = 0;
|
||||
*pause = 25 + i * 3; // 0.5-1 s apart, staggered
|
||||
}
|
||||
}
|
||||
}
|
||||
load.close();
|
||||
let n = *finals.lock().unwrap();
|
||||
n
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn conversation_round_trip_on_real_discord() {
|
||||
if std::env::var("JARVIS_LIVE_ROUND_TRIP").as_deref() != Ok("1") {
|
||||
eprintln!("skipped: set JARVIS_LIVE_ROUND_TRIP=1");
|
||||
return;
|
||||
}
|
||||
let guild: u64 = env("JARVIS_LIVE_GUILD").parse().unwrap();
|
||||
let voice: u64 = env("JARVIS_LIVE_VOICE_CHANNEL").parse().unwrap();
|
||||
let rounds: usize = std::env::var("JARVIS_LIVE_ROUNDS").ok().and_then(|v| v.parse().ok()).unwrap_or(2);
|
||||
let questions = PathBuf::from(env("JARVIS_LIVE_QUESTIONS"));
|
||||
env("JARVIS_CONFIG_PATH"); // the worker's brain reads it
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
|
||||
// The speaker bot first: it checks the channel is free before anything joins.
|
||||
let speaker_songbird = Songbird::serenity_from_config(Config::default().decode_mode(DecodeMode::Decrypt));
|
||||
let (rtx, mut rrx) = mpsc::unbounded_channel();
|
||||
let mut speaker = Client::builder(env("JARVIS_LIVE_SPEAKER_TOKEN"), GatewayIntents::GUILDS | GatewayIntents::GUILD_VOICE_STATES)
|
||||
.event_handler(ReadyOnce(rtx)).register_songbird_with(speaker_songbird.clone()).await.unwrap();
|
||||
let (s_cache, s_shards) = (speaker.cache.clone(), speaker.shard_manager.clone());
|
||||
tokio::spawn(async move { speaker.start().await.unwrap() });
|
||||
let speaker_id = rrx.recv().await.unwrap().user.id.get();
|
||||
let bot_token = env("JARVIS_LIVE_BOT_TOKEN");
|
||||
let jarvis_app_id = serenity::all::Http::new(&bot_token).get_current_application_info().await.unwrap().id.get();
|
||||
until("guild in cache", 10_000, || s_cache.guild(GuildId::new(guild)).is_some()).await;
|
||||
{
|
||||
let g = s_cache.guild(GuildId::new(guild)).unwrap();
|
||||
let people = g.voice_states.values().filter(|v| v.channel_id.is_some_and(|c| c.get() == voice)
|
||||
&& !s_cache.user(v.user_id).map(|u| u.bot).unwrap_or(false)).count();
|
||||
assert_eq!(people, 0, "refusing to test: people are in the voice channel");
|
||||
for id in [speaker_id, jarvis_app_id] {
|
||||
assert!(g.voice_states.get(&UserId::new(id)).and_then(|v| v.channel_id).is_none(),
|
||||
"refusing to test: bot {id} is in a voice channel");
|
||||
}
|
||||
}
|
||||
|
||||
// The app's services, as the app starts them, from a fresh data folder.
|
||||
let data = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("live-round-trip-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&data);
|
||||
std::fs::create_dir_all(data.join("config")).unwrap();
|
||||
let mut args = vec!["--history".to_string(), data.join("history.sqlite").to_string_lossy().into_owned(), "--brain".into()];
|
||||
if let Ok(m) = std::env::var("JARVIS_MELO_PYTHON") {
|
||||
args.extend(["--melo-python".to_string(), m]);
|
||||
}
|
||||
std::fs::write(data.join("config").join("settings.json"), serde_json::to_string_pretty(&json!({
|
||||
"discord": {"guildId": guild.to_string(), "voiceChannelId": voice.to_string()},
|
||||
// The "person" here is a bot, so bots must be heard.
|
||||
"voice": {"listenToBots": true},
|
||||
"voiceWorker": {"python": env("JARVIS_VOICE_PYTHON"), "cwd": root.join("../src"), "args": args},
|
||||
})).unwrap()).unwrap();
|
||||
std::env::set_var("JARVIS_DISCORD_TOKEN", &bot_token);
|
||||
let services = tokio::time::timeout(Duration::from_secs(300), start_services(&data)).await
|
||||
.expect("services start in time").expect("services start");
|
||||
services.worker.wait_ready(Duration::from_secs(240)).await.expect("worker ready");
|
||||
let core = services.core.clone();
|
||||
let events: Arc<Mutex<Vec<(Instant, CoreEvent)>>> = Arc::default();
|
||||
let ev2 = events.clone();
|
||||
let mut rx = services.events;
|
||||
tokio::spawn(async move {
|
||||
while let Some(e) = rx.recv().await {
|
||||
ev2.lock().unwrap().push((Instant::now(), e));
|
||||
}
|
||||
});
|
||||
until("Jarvis in voice", 20_000, || core.voice_channel_id() == Some(voice)).await;
|
||||
let jarvis = core.bot_id().unwrap();
|
||||
|
||||
let call = speaker_songbird.join(GuildId::new(guild), ChannelId::new(voice)).await.expect("speaker joins");
|
||||
let ears = Ears { jarvis, ..Default::default() };
|
||||
{
|
||||
let mut c = call.lock().await;
|
||||
c.add_global_event(VoiceEv::SpeakingStateUpdate.into(), ears.clone());
|
||||
c.add_global_event(VoiceEv::RtpPacket.into(), ears.clone());
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(2)).await; // the DAVE group settles after the join
|
||||
|
||||
let refs: Value = serde_json::from_str(&std::fs::read_to_string(questions.join("refs.json")).unwrap()).unwrap();
|
||||
let qs: Vec<(String, Vec<u8>)> = refs["samples"].as_array().unwrap().iter()
|
||||
.map(|r| (r["text"].as_str().unwrap().to_owned(), std::fs::read(questions.join(r["file"].as_str().unwrap())).unwrap()))
|
||||
.collect();
|
||||
let load_samples: Vec<Vec<i16>> = std::fs::read_dir(root.join("../scripts/m0/assets/ko_samples")).unwrap()
|
||||
.filter_map(|e| e.ok().map(|e| e.path())).filter(|p| p.extension().is_some_and(|x| x == "wav"))
|
||||
.map(|p| wav_samples(&std::fs::read(p).unwrap())).collect();
|
||||
|
||||
let mut report = serde_json::Map::new();
|
||||
let steps = futures_util::FutureExt::catch_unwind(std::panic::AssertUnwindSafe(async {
|
||||
for phase in ["alone", "with_9_more_voices"] {
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let load = (phase != "alone").then(|| {
|
||||
background_voices(services.worker_process.url(), services.worker_process.token.clone(), load_samples.clone(), stop.clone())
|
||||
});
|
||||
if load.is_some() {
|
||||
tokio::time::sleep(Duration::from_secs(3)).await;
|
||||
}
|
||||
let mut rows = vec![];
|
||||
for round in 0..rounds {
|
||||
for (text, wav) in &qs {
|
||||
// The previous answer is over before the next question.
|
||||
until("Jarvis to finish speaking", 60_000, || !core.speaking()).await;
|
||||
let before = events.lock().unwrap().len();
|
||||
*ears.first_audible.lock().unwrap() = None;
|
||||
let track = call.lock().await.play_only_input(Input::from(wav.clone()));
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
let started = loop {
|
||||
if track.get_info().await.is_ok_and(|i| i.position > Duration::ZERO) {
|
||||
break Instant::now();
|
||||
}
|
||||
assert!(Instant::now() < deadline, "timed out waiting for the track to play");
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
};
|
||||
let speech_end = started + Duration::from_secs_f64(speech_end_seconds(wav));
|
||||
// The answer is over when its own playback ends (or nothing came for 30 s).
|
||||
let finished = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
{
|
||||
let ev = events.lock().unwrap();
|
||||
let id = ev[before..].iter().find_map(|(_, e)| match e {
|
||||
CoreEvent::ReplyText { id, .. } => Some(id.clone()),
|
||||
_ => None,
|
||||
});
|
||||
if id.is_some_and(|id| ev[before..].iter().any(|(_, e)| matches!(e, CoreEvent::SpeakEnd(x) if x.id == id))) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
}).await.is_ok();
|
||||
let ev = events.lock().unwrap()[before..].to_vec();
|
||||
let ms = |t: Option<Instant>| t.map(|t| t.saturating_duration_since(speech_end).as_millis() as i64);
|
||||
let reply = ev.iter().find_map(|(t, e)| match e {
|
||||
CoreEvent::ReplyText { text, failed, .. } => Some((*t, text.clone(), *failed)),
|
||||
_ => None,
|
||||
});
|
||||
// A question may be heard as several finals (a pause after the wake word): all of
|
||||
// them are what was heard, and the last one before the answer is the one timed.
|
||||
let finals: Vec<_> = ev.iter().filter_map(|(t, e)| match e {
|
||||
CoreEvent::Final(f) if f.heard.speaker_id == speaker_id && !f.text.is_empty()
|
||||
&& reply.as_ref().is_none_or(|r| *t <= r.0) => Some((*t, f.clone())),
|
||||
_ => None,
|
||||
}).collect();
|
||||
let final_ev = finals.last().map(|(t, f)| (*t, HeardText {
|
||||
text: finals.iter().map(|(_, f)| f.text.as_str()).collect::<Vec<_>>().join(" | "),
|
||||
stt_ms: f.stt_ms,
|
||||
}));
|
||||
let speak_start = ev.iter().find_map(|(t, e)| matches!(e, CoreEvent::SpeakStart(_)).then_some(*t));
|
||||
let first_sound = *ears.first_audible.lock().unwrap();
|
||||
rows.push(json!({
|
||||
"round": round, "question": text,
|
||||
"heard": final_ev.as_ref().map(|(_, f)| f.text.clone()),
|
||||
"stt_ms": final_ev.as_ref().map(|(_, f)| f.stt_ms),
|
||||
"answer": reply.as_ref().map(|r| r.1.clone()),
|
||||
"final_ms": ms(final_ev.as_ref().map(|f| f.0)),
|
||||
"answer_text_ms": ms(reply.as_ref().map(|r| r.0)),
|
||||
"speak_start_ms": ms(speak_start),
|
||||
"first_sound_ms": ms(first_sound),
|
||||
"spoken": finished,
|
||||
}));
|
||||
println!("{phase} {}", rows.last().unwrap());
|
||||
tokio::time::sleep(Duration::from_millis(1500)).await;
|
||||
}
|
||||
}
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
let load_finals = match load { Some(h) => Some(h.await.unwrap()), None => None };
|
||||
let mut sound: Vec<i64> = rows.iter().filter_map(|r| r["first_sound_ms"].as_i64()).collect();
|
||||
sound.sort_unstable();
|
||||
let pick = |q: f64| sound.get(((sound.len() as f64 - 1.0) * q).round() as usize).copied();
|
||||
report.insert(phase.into(), json!({
|
||||
"rows": rows, "answered": sound.len(), "asked": rows.len(),
|
||||
"first_sound_ms": {"min": sound.first(), "median": pick(0.5), "p90": pick(0.9), "max": sound.last()},
|
||||
"background_finals": load_finals,
|
||||
}));
|
||||
}
|
||||
println!("live round trip {}", serde_json::to_string_pretty(&Value::Object(report.clone())).unwrap());
|
||||
})).await;
|
||||
|
||||
let _ = speaker_songbird.remove(GuildId::new(guild)).await;
|
||||
s_shards.shutdown_all().await;
|
||||
services.core.leave("test").await;
|
||||
services.core.close().await;
|
||||
services.worker.close();
|
||||
services.worker_process.stop().await;
|
||||
if let Err(panic) = steps {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
// Every question is answered aloud.
|
||||
for phase in ["alone", "with_9_more_voices"] {
|
||||
let p = &report[phase];
|
||||
assert_eq!(p["answered"], p["asked"], "{phase}: not every question got a spoken answer: {p}");
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user