test(app): time the whole conversation on real Discord, alone and with 9 more voices

live_round_trip.rs runs the app's services with the real worker and brain,
plays recorded questions from a second bot and times the final, the
answer text, playback start and the first audible frame after the speech
ends; a second worker session streams nine more voices for the load run.
Roadmap: first M4 step 4 numbers (median 3.5 s alone, 4.4 s under load).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-12 09:58:55 +09:00
parent bd2eb49e3d
commit 5507ec9b71
2 changed files with 355 additions and 1 deletions

View File

@@ -0,0 +1,354 @@
//! M4 step 4: the whole conversation on real Discord, timed from the end of a question to Jarvis's
//! first sound, with the real voice worker, the real brain (Strata) and the real response policy.
//!
//! The "speaker" bot plays recorded Korean questions that call Jarvis into the voice channel and
//! listens for the answer. The Jarvis bot runs the app's services (`start_services`: core,
//! conversation loop, worker process). Each question is timed at four points after the speech ends:
//! the final transcript, the brain's answer text, playback start and the first audible frame the
//! speaker bot receives. The rounds run again while nine more voices talk at the same time (a second
//! worker session feeding the shared STT), which is the 10-person load.
//! Opt-in; it refuses to run if a person is in the channel or either bot is already in voice.
//!
//! JARVIS_LIVE_ROUND_TRIP=1 JARVIS_LIVE_BOT_TOKEN=... JARVIS_LIVE_SPEAKER_TOKEN=... \
//! JARVIS_LIVE_GUILD=<server id> JARVIS_LIVE_VOICE_CHANNEL=<voice channel id> \
//! JARVIS_VOICE_PYTHON=<worker venv python> JARVIS_MELO_PYTHON=<MeloTTS venv python> \
//! JARVIS_CONFIG_PATH=<jarvis settings whose ollama_base_url is Strata's /v1> \
//! JARVIS_LIVE_QUESTIONS=<dir with refs.json and 16 kHz mono wavs> [JARVIS_LIVE_ROUNDS=2] \
//! cargo test --release --test live_round_trip -- --nocapture
//!
//! On Linux, put the pip CUDA libraries on LD_LIBRARY_PATH (nvidia/cublas/lib, nvidia/cudnn/lib).
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use async_trait::async_trait;
use jarvis_app::discord::CoreEvent;
use jarvis_app::receive::{opus_payload, OpusFrames, SpeakerStream};
use jarvis_app::services::start_services;
use jarvis_app::worker_client::{WorkerClient, WorkerClientOptions};
use serde_json::{json, Value};
use serenity::all::{ChannelId, Client, Context, EventHandler, GatewayIntents, GuildId, Ready, UserId};
use songbird::driver::DecodeMode;
use songbird::input::Input;
use songbird::{Config, CoreEvent as VoiceEv, Event, EventContext, EventHandler as VoiceHandler, SerenityInit, Songbird};
use tokio::sync::mpsc;
fn env(k: &str) -> String {
std::env::var(k).unwrap_or_else(|_| panic!("{k} is not set"))
}
/// Seconds from the start of a 16 kHz mono PCM16 wav to its last sample above -34 dBFS.
fn speech_end_seconds(wav: &[u8]) -> f64 {
let data = &wav[44..];
let n = data.len() / 2;
(0..n).rev().find(|&k| i16::from_le_bytes([data[2 * k], data[2 * k + 1]]).unsigned_abs() > 655)
.map_or(0.0, |k| (k + 1) as f64 / 16000.0)
}
fn wav_samples(wav: &[u8]) -> Vec<i16> {
wav[44..].chunks_exact(2).map(|b| i16::from_le_bytes([b[0], b[1]])).collect()
}
struct HeardText {
text: String,
stt_ms: u64,
}
struct ReadyOnce(mpsc::UnboundedSender<Ready>);
#[async_trait]
impl EventHandler for ReadyOnce {
async fn ready(&self, _: Context, r: Ready) {
let _ = self.0.send(r);
}
}
/// When Jarvis's voice first becomes audible to the speaker bot, decoded the way the core decodes
/// (songbird's own decoder garbles DAVE audio, see src/receive.rs).
#[derive(Clone, Default)]
struct Ears {
jarvis: u64,
ssrc: Arc<Mutex<HashMap<u32, u64>>>,
stream: Arc<Mutex<Option<SpeakerStream<OpusFrames>>>>,
first_audible: Arc<Mutex<Option<Instant>>>,
}
#[async_trait]
impl VoiceHandler for Ears {
async fn act(&self, ctx: &EventContext<'_>) -> Option<Event> {
match ctx {
EventContext::SpeakingStateUpdate(s) => {
if let Some(u) = s.user_id {
self.ssrc.lock().unwrap().insert(s.ssrc, u.0);
}
}
EventContext::RtpPacket(p) => {
let (seq, ssrc, opus) = opus_payload(&p.packet, p.payload_offset, p.payload_end_pad)?;
if self.ssrc.lock().unwrap().get(&ssrc) != Some(&self.jarvis) {
return None;
}
let frames = self.stream.lock().unwrap().get_or_insert_with(|| SpeakerStream::new(OpusFrames::new()))
.push(seq, opus.to_vec());
for f in frames {
// Audible: louder than -36 dBFS. Silence frames do not count.
let rms = (f.iter().map(|&x| f64::from(x).powi(2)).sum::<f64>() / f.len().max(1) as f64).sqrt();
if rms > 500.0 {
self.first_audible.lock().unwrap().get_or_insert_with(Instant::now);
}
}
}
_ => {}
}
None
}
}
async fn until(what: &str, ms: u64, mut pred: impl FnMut() -> bool) {
let deadline = Instant::now() + Duration::from_millis(ms);
while !pred() {
assert!(Instant::now() < deadline, "timed out waiting for {what}");
tokio::time::sleep(Duration::from_millis(10)).await;
}
}
/// Nine more voices: a second worker session streams recorded sentences for nine speakers in real
/// time, all talking at once with short pauses, until `stop` is set.
fn background_voices(url: String, token: String, samples: Vec<Vec<i16>>, stop: Arc<AtomicBool>) -> tokio::task::JoinHandle<u64> {
tokio::spawn(async move {
let mut o = WorkerClientOptions::new(url, token);
o.stt = json!({"partials": true}).as_object().unwrap().clone();
let (load, mut rx) = WorkerClient::new(o);
load.start();
load.wait_ready(Duration::from_secs(10)).await.expect("load session hello");
let finals = Arc::new(Mutex::new(0u64));
let f2 = finals.clone();
tokio::spawn(async move {
while let Some(e) = rx.recv().await {
if matches!(e, jarvis_app::worker_client::WorkerEvent::Final(_)) {
*f2.lock().unwrap() += 1;
}
}
});
// 20 ms of 48 kHz stereo per frame, the format the core sends.
let mut pos: Vec<(usize, usize, usize)> = (0..9).map(|i| (i % samples.len(), i * 4000, 0)).collect();
let mut tick = tokio::time::interval(Duration::from_millis(20));
while !stop.load(Ordering::Relaxed) {
tick.tick().await;
for (i, (sample, at, pause)) in pos.iter_mut().enumerate() {
let speaker = format!("load-{i}");
if *pause > 0 {
*pause -= 1;
if *pause == 0 {
load.speaking_end(&speaker);
}
continue;
}
let s = &samples[*sample];
let end = (*at + 320).min(s.len());
let frame: Vec<i16> = s[*at..end].iter().flat_map(|&x| [x; 6]).collect();
load.send_audio(&speaker, &frame);
*at = end;
if end == s.len() {
*sample = (*sample + 1) % samples.len();
*at = 0;
*pause = 25 + i * 3; // 0.5-1 s apart, staggered
}
}
}
load.close();
let n = *finals.lock().unwrap();
n
})
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn conversation_round_trip_on_real_discord() {
if std::env::var("JARVIS_LIVE_ROUND_TRIP").as_deref() != Ok("1") {
eprintln!("skipped: set JARVIS_LIVE_ROUND_TRIP=1");
return;
}
let guild: u64 = env("JARVIS_LIVE_GUILD").parse().unwrap();
let voice: u64 = env("JARVIS_LIVE_VOICE_CHANNEL").parse().unwrap();
let rounds: usize = std::env::var("JARVIS_LIVE_ROUNDS").ok().and_then(|v| v.parse().ok()).unwrap_or(2);
let questions = PathBuf::from(env("JARVIS_LIVE_QUESTIONS"));
env("JARVIS_CONFIG_PATH"); // the worker's brain reads it
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
// The speaker bot first: it checks the channel is free before anything joins.
let speaker_songbird = Songbird::serenity_from_config(Config::default().decode_mode(DecodeMode::Decrypt));
let (rtx, mut rrx) = mpsc::unbounded_channel();
let mut speaker = Client::builder(env("JARVIS_LIVE_SPEAKER_TOKEN"), GatewayIntents::GUILDS | GatewayIntents::GUILD_VOICE_STATES)
.event_handler(ReadyOnce(rtx)).register_songbird_with(speaker_songbird.clone()).await.unwrap();
let (s_cache, s_shards) = (speaker.cache.clone(), speaker.shard_manager.clone());
tokio::spawn(async move { speaker.start().await.unwrap() });
let speaker_id = rrx.recv().await.unwrap().user.id.get();
let bot_token = env("JARVIS_LIVE_BOT_TOKEN");
let jarvis_app_id = serenity::all::Http::new(&bot_token).get_current_application_info().await.unwrap().id.get();
until("guild in cache", 10_000, || s_cache.guild(GuildId::new(guild)).is_some()).await;
{
let g = s_cache.guild(GuildId::new(guild)).unwrap();
let people = g.voice_states.values().filter(|v| v.channel_id.is_some_and(|c| c.get() == voice)
&& !s_cache.user(v.user_id).map(|u| u.bot).unwrap_or(false)).count();
assert_eq!(people, 0, "refusing to test: people are in the voice channel");
for id in [speaker_id, jarvis_app_id] {
assert!(g.voice_states.get(&UserId::new(id)).and_then(|v| v.channel_id).is_none(),
"refusing to test: bot {id} is in a voice channel");
}
}
// The app's services, as the app starts them, from a fresh data folder.
let data = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("live-round-trip-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&data);
std::fs::create_dir_all(data.join("config")).unwrap();
let mut args = vec!["--history".to_string(), data.join("history.sqlite").to_string_lossy().into_owned(), "--brain".into()];
if let Ok(m) = std::env::var("JARVIS_MELO_PYTHON") {
args.extend(["--melo-python".to_string(), m]);
}
std::fs::write(data.join("config").join("settings.json"), serde_json::to_string_pretty(&json!({
"discord": {"guildId": guild.to_string(), "voiceChannelId": voice.to_string()},
// The "person" here is a bot, so bots must be heard.
"voice": {"listenToBots": true},
"voiceWorker": {"python": env("JARVIS_VOICE_PYTHON"), "cwd": root.join("../src"), "args": args},
})).unwrap()).unwrap();
std::env::set_var("JARVIS_DISCORD_TOKEN", &bot_token);
let services = tokio::time::timeout(Duration::from_secs(300), start_services(&data)).await
.expect("services start in time").expect("services start");
services.worker.wait_ready(Duration::from_secs(240)).await.expect("worker ready");
let core = services.core.clone();
let events: Arc<Mutex<Vec<(Instant, CoreEvent)>>> = Arc::default();
let ev2 = events.clone();
let mut rx = services.events;
tokio::spawn(async move {
while let Some(e) = rx.recv().await {
ev2.lock().unwrap().push((Instant::now(), e));
}
});
until("Jarvis in voice", 20_000, || core.voice_channel_id() == Some(voice)).await;
let jarvis = core.bot_id().unwrap();
let call = speaker_songbird.join(GuildId::new(guild), ChannelId::new(voice)).await.expect("speaker joins");
let ears = Ears { jarvis, ..Default::default() };
{
let mut c = call.lock().await;
c.add_global_event(VoiceEv::SpeakingStateUpdate.into(), ears.clone());
c.add_global_event(VoiceEv::RtpPacket.into(), ears.clone());
}
tokio::time::sleep(Duration::from_secs(2)).await; // the DAVE group settles after the join
let refs: Value = serde_json::from_str(&std::fs::read_to_string(questions.join("refs.json")).unwrap()).unwrap();
let qs: Vec<(String, Vec<u8>)> = refs["samples"].as_array().unwrap().iter()
.map(|r| (r["text"].as_str().unwrap().to_owned(), std::fs::read(questions.join(r["file"].as_str().unwrap())).unwrap()))
.collect();
let load_samples: Vec<Vec<i16>> = std::fs::read_dir(root.join("../scripts/m0/assets/ko_samples")).unwrap()
.filter_map(|e| e.ok().map(|e| e.path())).filter(|p| p.extension().is_some_and(|x| x == "wav"))
.map(|p| wav_samples(&std::fs::read(p).unwrap())).collect();
let mut report = serde_json::Map::new();
let steps = futures_util::FutureExt::catch_unwind(std::panic::AssertUnwindSafe(async {
for phase in ["alone", "with_9_more_voices"] {
let stop = Arc::new(AtomicBool::new(false));
let load = (phase != "alone").then(|| {
background_voices(services.worker_process.url(), services.worker_process.token.clone(), load_samples.clone(), stop.clone())
});
if load.is_some() {
tokio::time::sleep(Duration::from_secs(3)).await;
}
let mut rows = vec![];
for round in 0..rounds {
for (text, wav) in &qs {
// The previous answer is over before the next question.
until("Jarvis to finish speaking", 60_000, || !core.speaking()).await;
let before = events.lock().unwrap().len();
*ears.first_audible.lock().unwrap() = None;
let track = call.lock().await.play_only_input(Input::from(wav.clone()));
let deadline = Instant::now() + Duration::from_secs(5);
let started = loop {
if track.get_info().await.is_ok_and(|i| i.position > Duration::ZERO) {
break Instant::now();
}
assert!(Instant::now() < deadline, "timed out waiting for the track to play");
tokio::time::sleep(Duration::from_millis(5)).await;
};
let speech_end = started + Duration::from_secs_f64(speech_end_seconds(wav));
// The answer is over when its own playback ends (or nothing came for 30 s).
let finished = tokio::time::timeout(Duration::from_secs(30), async {
loop {
{
let ev = events.lock().unwrap();
let id = ev[before..].iter().find_map(|(_, e)| match e {
CoreEvent::ReplyText { id, .. } => Some(id.clone()),
_ => None,
});
if id.is_some_and(|id| ev[before..].iter().any(|(_, e)| matches!(e, CoreEvent::SpeakEnd(x) if x.id == id))) {
return;
}
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
}).await.is_ok();
let ev = events.lock().unwrap()[before..].to_vec();
let ms = |t: Option<Instant>| t.map(|t| t.saturating_duration_since(speech_end).as_millis() as i64);
let reply = ev.iter().find_map(|(t, e)| match e {
CoreEvent::ReplyText { text, failed, .. } => Some((*t, text.clone(), *failed)),
_ => None,
});
// A question may be heard as several finals (a pause after the wake word): all of
// them are what was heard, and the last one before the answer is the one timed.
let finals: Vec<_> = ev.iter().filter_map(|(t, e)| match e {
CoreEvent::Final(f) if f.heard.speaker_id == speaker_id && !f.text.is_empty()
&& reply.as_ref().is_none_or(|r| *t <= r.0) => Some((*t, f.clone())),
_ => None,
}).collect();
let final_ev = finals.last().map(|(t, f)| (*t, HeardText {
text: finals.iter().map(|(_, f)| f.text.as_str()).collect::<Vec<_>>().join(" | "),
stt_ms: f.stt_ms,
}));
let speak_start = ev.iter().find_map(|(t, e)| matches!(e, CoreEvent::SpeakStart(_)).then_some(*t));
let first_sound = *ears.first_audible.lock().unwrap();
rows.push(json!({
"round": round, "question": text,
"heard": final_ev.as_ref().map(|(_, f)| f.text.clone()),
"stt_ms": final_ev.as_ref().map(|(_, f)| f.stt_ms),
"answer": reply.as_ref().map(|r| r.1.clone()),
"final_ms": ms(final_ev.as_ref().map(|f| f.0)),
"answer_text_ms": ms(reply.as_ref().map(|r| r.0)),
"speak_start_ms": ms(speak_start),
"first_sound_ms": ms(first_sound),
"spoken": finished,
}));
println!("{phase} {}", rows.last().unwrap());
tokio::time::sleep(Duration::from_millis(1500)).await;
}
}
stop.store(true, Ordering::Relaxed);
let load_finals = match load { Some(h) => Some(h.await.unwrap()), None => None };
let mut sound: Vec<i64> = rows.iter().filter_map(|r| r["first_sound_ms"].as_i64()).collect();
sound.sort_unstable();
let pick = |q: f64| sound.get(((sound.len() as f64 - 1.0) * q).round() as usize).copied();
report.insert(phase.into(), json!({
"rows": rows, "answered": sound.len(), "asked": rows.len(),
"first_sound_ms": {"min": sound.first(), "median": pick(0.5), "p90": pick(0.9), "max": sound.last()},
"background_finals": load_finals,
}));
}
println!("live round trip {}", serde_json::to_string_pretty(&Value::Object(report.clone())).unwrap());
})).await;
let _ = speaker_songbird.remove(GuildId::new(guild)).await;
s_shards.shutdown_all().await;
services.core.leave("test").await;
services.core.close().await;
services.worker.close();
services.worker_process.stop().await;
if let Err(panic) = steps {
std::panic::resume_unwind(panic);
}
// Every question is answered aloud.
for phase in ["alone", "with_9_more_voices"] {
let p = &report[phase];
assert_eq!(p["answered"], p["asked"], "{phase}: not every question got a spoken answer: {p}");
}
}

File diff suppressed because one or more lines are too long