M5 step 4, second part. The services start the LLM first: the installed Strata with the M0 kit's model and normal placement (models in models/llm, parallel 1, or 2 when a chat channel answers every message, pictures off, --restart), wait until /v1/models lists the model, and stop it last. The first start downloads the model; the dashboard shows Strata's last line and percentage, and stop also cancels a start (the server is cleared). llm.url uses a server the app does not start. On every start the app writes config/brain.json (Strata's /v1, the listed model id, memory/jarvis.db) and hands it to the worker as JARVIS_CONFIG_PATH. Loading a config with its own db_path no longer creates ~/.local/share/jarvis. service_start/service_restart run in the background. Settings gain llm.context and llm.port (model screen) and llm.url (file only). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
374 lines
19 KiB
Rust
374 lines
19 KiB
Rust
//! M4 step 4: the whole conversation on real Discord, timed from the end of a question to Jarvis's
|
|
//! first sound, with the real voice worker, the real brain (Strata) and the real response policy.
|
|
//!
|
|
//! The "speaker" bot plays recorded Korean questions that call Jarvis into the voice channel and
|
|
//! listens for the answer. The Jarvis bot runs the app's services (`start_services`: core,
|
|
//! conversation loop, worker process). Each question is timed at four points after the speech ends:
|
|
//! the final transcript, the brain's answer text, playback start and the first audible frame the
|
|
//! speaker bot receives. The rounds run again while nine more voices talk at the same time (a second
|
|
//! worker session feeding the shared STT), which is the 10-person load.
|
|
//! Opt-in; it refuses to run if a person is in the channel or either bot is already in voice.
|
|
//!
|
|
//! JARVIS_LIVE_ROUND_TRIP=1 JARVIS_LIVE_BOT_TOKEN=... JARVIS_LIVE_SPEAKER_TOKEN=... \
|
|
//! JARVIS_LIVE_GUILD=<server id> JARVIS_LIVE_VOICE_CHANNEL=<voice channel id> \
|
|
//! JARVIS_VOICE_PYTHON=<worker venv python> JARVIS_MELO_PYTHON=<MeloTTS venv python> \
|
|
//! JARVIS_LIVE_LLM_URL=<Strata's /v1 address, e.g. a tunnel to the game PC> \
|
|
//! JARVIS_LIVE_QUESTIONS=<dir with refs.json and 16 kHz mono wavs> [JARVIS_LIVE_ROUNDS=2] \
|
|
//! cargo test --release --test live_round_trip -- --nocapture
|
|
//!
|
|
//! On Linux, put the pip CUDA libraries on LD_LIBRARY_PATH (nvidia/cublas/lib, nvidia/cudnn/lib).
|
|
|
|
use std::collections::HashMap;
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::{Duration, Instant};
|
|
|
|
use async_trait::async_trait;
|
|
use jarvis_app::discord::CoreEvent;
|
|
use jarvis_app::receive::{opus_payload, OpusFrames, SpeakerStream};
|
|
use jarvis_app::services::start_services;
|
|
use jarvis_app::strata::Llm;
|
|
use jarvis_app::worker_client::{WorkerClient, WorkerClientOptions};
|
|
use serde_json::{json, Value};
|
|
use serenity::all::{ChannelId, Client, Context, EventHandler, GatewayIntents, GuildId, Ready, UserId};
|
|
use songbird::driver::DecodeMode;
|
|
use songbird::input::Input;
|
|
use songbird::{Config, CoreEvent as VoiceEv, Event, EventContext, EventHandler as VoiceHandler, SerenityInit, Songbird};
|
|
use tokio::sync::mpsc;
|
|
|
|
fn env(k: &str) -> String {
|
|
std::env::var(k).unwrap_or_else(|_| panic!("{k} is not set"))
|
|
}
|
|
|
|
/// Seconds from the start of a 16 kHz mono PCM16 wav to its last sample above -34 dBFS.
|
|
fn speech_end_seconds(wav: &[u8]) -> f64 {
|
|
let data = &wav[44..];
|
|
let n = data.len() / 2;
|
|
(0..n).rev().find(|&k| i16::from_le_bytes([data[2 * k], data[2 * k + 1]]).unsigned_abs() > 655)
|
|
.map_or(0.0, |k| (k + 1) as f64 / 16000.0)
|
|
}
|
|
|
|
fn wav_samples(wav: &[u8]) -> Vec<i16> {
|
|
wav[44..].chunks_exact(2).map(|b| i16::from_le_bytes([b[0], b[1]])).collect()
|
|
}
|
|
|
|
struct HeardText {
|
|
text: String,
|
|
stt_ms: u64,
|
|
}
|
|
|
|
struct ReadyOnce(mpsc::UnboundedSender<Ready>);
|
|
#[async_trait]
|
|
impl EventHandler for ReadyOnce {
|
|
async fn ready(&self, _: Context, r: Ready) {
|
|
let _ = self.0.send(r);
|
|
}
|
|
}
|
|
|
|
/// When Jarvis's voice first becomes audible to the speaker bot, decoded the way the core decodes
|
|
/// (songbird's own decoder garbles DAVE audio, see src/receive.rs).
|
|
#[derive(Clone, Default)]
|
|
struct Ears {
|
|
jarvis: u64,
|
|
ssrc: Arc<Mutex<HashMap<u32, u64>>>,
|
|
stream: Arc<Mutex<Option<SpeakerStream<OpusFrames>>>>,
|
|
/// When each audible 20 ms frame arrived (the earcon first, then the answer).
|
|
audible: Arc<Mutex<Vec<Instant>>>,
|
|
}
|
|
|
|
#[async_trait]
|
|
impl VoiceHandler for Ears {
|
|
async fn act(&self, ctx: &EventContext<'_>) -> Option<Event> {
|
|
match ctx {
|
|
EventContext::SpeakingStateUpdate(s) => {
|
|
if let Some(u) = s.user_id {
|
|
self.ssrc.lock().unwrap().insert(s.ssrc, u.0);
|
|
}
|
|
}
|
|
EventContext::RtpPacket(p) => {
|
|
let (seq, ssrc, opus) = opus_payload(&p.packet, p.payload_offset, p.payload_end_pad)?;
|
|
if self.ssrc.lock().unwrap().get(&ssrc) != Some(&self.jarvis) {
|
|
return None;
|
|
}
|
|
let frames = self.stream.lock().unwrap().get_or_insert_with(|| SpeakerStream::new(OpusFrames::new()))
|
|
.push(seq, opus.to_vec());
|
|
for f in frames {
|
|
// Audible: louder than -36 dBFS. Silence frames do not count.
|
|
let rms = (f.iter().map(|&x| f64::from(x).powi(2)).sum::<f64>() / f.len().max(1) as f64).sqrt();
|
|
if rms > 500.0 {
|
|
self.audible.lock().unwrap().push(Instant::now());
|
|
}
|
|
}
|
|
}
|
|
_ => {}
|
|
}
|
|
None
|
|
}
|
|
}
|
|
|
|
async fn until(what: &str, ms: u64, mut pred: impl FnMut() -> bool) {
|
|
let deadline = Instant::now() + Duration::from_millis(ms);
|
|
while !pred() {
|
|
assert!(Instant::now() < deadline, "timed out waiting for {what}");
|
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
|
}
|
|
}
|
|
|
|
/// Nine more voices: a second worker session streams recorded sentences for nine speakers in real
|
|
/// time, all talking at once with short pauses, until `stop` is set.
|
|
fn background_voices(url: String, token: String, samples: Vec<Vec<i16>>, stop: Arc<AtomicBool>) -> tokio::task::JoinHandle<u64> {
|
|
tokio::spawn(async move {
|
|
let mut o = WorkerClientOptions::new(url, token);
|
|
o.stt = json!({"partials": true}).as_object().unwrap().clone();
|
|
let (load, mut rx) = WorkerClient::new(o);
|
|
load.start();
|
|
load.wait_ready(Duration::from_secs(10)).await.expect("load session hello");
|
|
let finals = Arc::new(Mutex::new(0u64));
|
|
let f2 = finals.clone();
|
|
tokio::spawn(async move {
|
|
while let Some(e) = rx.recv().await {
|
|
if matches!(e, jarvis_app::worker_client::WorkerEvent::Final(_)) {
|
|
*f2.lock().unwrap() += 1;
|
|
}
|
|
}
|
|
});
|
|
// 20 ms of 48 kHz stereo per frame, the format the core sends.
|
|
let mut pos: Vec<(usize, usize, usize)> = (0..9).map(|i| (i % samples.len(), i * 4000, 0)).collect();
|
|
let mut tick = tokio::time::interval(Duration::from_millis(20));
|
|
while !stop.load(Ordering::Relaxed) {
|
|
tick.tick().await;
|
|
for (i, (sample, at, pause)) in pos.iter_mut().enumerate() {
|
|
let speaker = format!("load-{i}");
|
|
if *pause > 0 {
|
|
*pause -= 1;
|
|
if *pause == 0 {
|
|
load.speaking_end(&speaker);
|
|
}
|
|
continue;
|
|
}
|
|
let s = &samples[*sample];
|
|
let end = (*at + 320).min(s.len());
|
|
let frame: Vec<i16> = s[*at..end].iter().flat_map(|&x| [x; 6]).collect();
|
|
load.send_audio(&speaker, &frame);
|
|
*at = end;
|
|
if end == s.len() {
|
|
*sample = (*sample + 1) % samples.len();
|
|
*at = 0;
|
|
*pause = 25 + i * 3; // 0.5-1 s apart, staggered
|
|
}
|
|
}
|
|
}
|
|
load.close();
|
|
let n = *finals.lock().unwrap();
|
|
n
|
|
})
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
|
async fn conversation_round_trip_on_real_discord() {
|
|
if std::env::var("JARVIS_LIVE_ROUND_TRIP").as_deref() != Ok("1") {
|
|
eprintln!("skipped: set JARVIS_LIVE_ROUND_TRIP=1");
|
|
return;
|
|
}
|
|
let guild: u64 = env("JARVIS_LIVE_GUILD").parse().unwrap();
|
|
let voice: u64 = env("JARVIS_LIVE_VOICE_CHANNEL").parse().unwrap();
|
|
let rounds: usize = std::env::var("JARVIS_LIVE_ROUNDS").ok().and_then(|v| v.parse().ok()).unwrap_or(2);
|
|
let questions = PathBuf::from(env("JARVIS_LIVE_QUESTIONS"));
|
|
env("JARVIS_LIVE_LLM_URL"); // the services point the brain at it (llm.url)
|
|
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
|
|
|
// The speaker bot first: it checks the channel is free before anything joins.
|
|
let speaker_songbird = Songbird::serenity_from_config(Config::default().decode_mode(DecodeMode::Decrypt));
|
|
let (rtx, mut rrx) = mpsc::unbounded_channel();
|
|
let mut speaker = Client::builder(env("JARVIS_LIVE_SPEAKER_TOKEN"), GatewayIntents::GUILDS | GatewayIntents::GUILD_VOICE_STATES)
|
|
.event_handler(ReadyOnce(rtx)).register_songbird_with(speaker_songbird.clone()).await.unwrap();
|
|
let (s_cache, s_shards) = (speaker.cache.clone(), speaker.shard_manager.clone());
|
|
tokio::spawn(async move { speaker.start().await.unwrap() });
|
|
let speaker_id = rrx.recv().await.unwrap().user.id.get();
|
|
let bot_token = env("JARVIS_LIVE_BOT_TOKEN");
|
|
let jarvis_app_id = serenity::all::Http::new(&bot_token).get_current_application_info().await.unwrap().id.get();
|
|
until("guild in cache", 10_000, || s_cache.guild(GuildId::new(guild)).is_some()).await;
|
|
{
|
|
let g = s_cache.guild(GuildId::new(guild)).unwrap();
|
|
let people = g.voice_states.values().filter(|v| v.channel_id.is_some_and(|c| c.get() == voice)
|
|
&& !s_cache.user(v.user_id).map(|u| u.bot).unwrap_or(false)).count();
|
|
assert_eq!(people, 0, "refusing to test: people are in the voice channel");
|
|
for id in [speaker_id, jarvis_app_id] {
|
|
assert!(g.voice_states.get(&UserId::new(id)).and_then(|v| v.channel_id).is_none(),
|
|
"refusing to test: bot {id} is in a voice channel");
|
|
}
|
|
}
|
|
|
|
// The app's services, as the app starts them, from a fresh data folder.
|
|
let data = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("live-round-trip-{}", std::process::id()));
|
|
let _ = std::fs::remove_dir_all(&data);
|
|
std::fs::create_dir_all(data.join("config")).unwrap();
|
|
let mut args = vec!["--history".to_string(), data.join("history.sqlite").to_string_lossy().into_owned(), "--brain".into()];
|
|
if let Ok(m) = std::env::var("JARVIS_MELO_PYTHON") {
|
|
args.extend(["--melo-python".to_string(), m]);
|
|
}
|
|
std::fs::write(data.join("config").join("settings.json"), serde_json::to_string_pretty(&json!({
|
|
"discord": {"guildId": guild.to_string(), "voiceChannelId": voice.to_string()},
|
|
// The "person" here is a bot, so bots must be heard.
|
|
"voice": {"listenToBots": true},
|
|
"voiceWorker": {"python": env("JARVIS_VOICE_PYTHON"), "cwd": root.join("../src"), "args": args},
|
|
"llm": {"url": env("JARVIS_LIVE_LLM_URL")},
|
|
})).unwrap()).unwrap();
|
|
std::env::set_var("JARVIS_DISCORD_TOKEN", &bot_token);
|
|
let services = tokio::time::timeout(Duration::from_secs(300), start_services(&data, Arc::new(Llm::new(&data)))).await
|
|
.expect("services start in time").expect("services start");
|
|
services.worker.wait_ready(Duration::from_secs(240)).await.expect("worker ready");
|
|
let core = services.core.clone();
|
|
let events: Arc<Mutex<Vec<(Instant, CoreEvent)>>> = Arc::default();
|
|
let ev2 = events.clone();
|
|
let mut rx = services.events;
|
|
tokio::spawn(async move {
|
|
while let Some(e) = rx.recv().await {
|
|
ev2.lock().unwrap().push((Instant::now(), e));
|
|
}
|
|
});
|
|
until("Jarvis in voice", 20_000, || core.voice_channel_id() == Some(voice)).await;
|
|
let jarvis = core.bot_id().unwrap();
|
|
|
|
let call = speaker_songbird.join(GuildId::new(guild), ChannelId::new(voice)).await.expect("speaker joins");
|
|
let ears = Ears { jarvis, ..Default::default() };
|
|
{
|
|
let mut c = call.lock().await;
|
|
c.add_global_event(VoiceEv::SpeakingStateUpdate.into(), ears.clone());
|
|
c.add_global_event(VoiceEv::RtpPacket.into(), ears.clone());
|
|
}
|
|
tokio::time::sleep(Duration::from_secs(2)).await; // the DAVE group settles after the join
|
|
|
|
let refs: Value = serde_json::from_str(&std::fs::read_to_string(questions.join("refs.json")).unwrap()).unwrap();
|
|
let qs: Vec<(String, Vec<u8>)> = refs["samples"].as_array().unwrap().iter()
|
|
.map(|r| (r["text"].as_str().unwrap().to_owned(), std::fs::read(questions.join(r["file"].as_str().unwrap())).unwrap()))
|
|
.collect();
|
|
let load_samples: Vec<Vec<i16>> = std::fs::read_dir(root.join("../scripts/m0/assets/ko_samples")).unwrap()
|
|
.filter_map(|e| e.ok().map(|e| e.path())).filter(|p| p.extension().is_some_and(|x| x == "wav"))
|
|
.map(|p| wav_samples(&std::fs::read(p).unwrap())).collect();
|
|
|
|
let mut report = serde_json::Map::new();
|
|
let steps = futures_util::FutureExt::catch_unwind(std::panic::AssertUnwindSafe(async {
|
|
for phase in ["alone", "with_9_more_voices"] {
|
|
let stop = Arc::new(AtomicBool::new(false));
|
|
let load = (phase != "alone").then(|| {
|
|
background_voices(services.worker_process.url(), services.worker_process.token.clone(), load_samples.clone(), stop.clone())
|
|
});
|
|
if load.is_some() {
|
|
tokio::time::sleep(Duration::from_secs(3)).await;
|
|
}
|
|
let mut rows = vec![];
|
|
for round in 0..rounds {
|
|
for (text, wav) in &qs {
|
|
// The previous answer is over before the next question.
|
|
until("Jarvis to finish speaking", 60_000, || !core.speaking()).await;
|
|
let before = events.lock().unwrap().len();
|
|
ears.audible.lock().unwrap().clear();
|
|
let track = call.lock().await.play_only_input(Input::from(wav.clone()));
|
|
let deadline = Instant::now() + Duration::from_secs(5);
|
|
let started = loop {
|
|
if track.get_info().await.is_ok_and(|i| i.position > Duration::ZERO) {
|
|
break Instant::now();
|
|
}
|
|
assert!(Instant::now() < deadline, "timed out waiting for the track to play");
|
|
tokio::time::sleep(Duration::from_millis(5)).await;
|
|
};
|
|
let speech_end = started + Duration::from_secs_f64(speech_end_seconds(wav));
|
|
// The answer is over when its own playback ends (or nothing came for 30 s).
|
|
let finished = tokio::time::timeout(Duration::from_secs(30), async {
|
|
loop {
|
|
{
|
|
let ev = events.lock().unwrap();
|
|
let id = ev[before..].iter().find_map(|(_, e)| match e {
|
|
CoreEvent::ReplyText { id, .. } => Some(id.clone()),
|
|
_ => None,
|
|
});
|
|
if id.is_some_and(|id| ev[before..].iter().any(|(_, e)| matches!(e, CoreEvent::SpeakEnd(x) if x.id == id))) {
|
|
return;
|
|
}
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(20)).await;
|
|
}
|
|
}).await.is_ok();
|
|
let ev = events.lock().unwrap()[before..].to_vec();
|
|
let ms = |t: Option<Instant>| t.map(|t| t.saturating_duration_since(speech_end).as_millis() as i64);
|
|
let reply = ev.iter().find_map(|(t, e)| match e {
|
|
CoreEvent::ReplyText { text, failed, .. } => Some((*t, text.clone(), *failed)),
|
|
_ => None,
|
|
});
|
|
// A question may be heard as several finals (a pause after the wake word): all of
|
|
// them are what was heard, and the last one before the answer is the one timed.
|
|
let finals: Vec<_> = ev.iter().filter_map(|(t, e)| match e {
|
|
CoreEvent::Final(f) if f.heard.speaker_id == speaker_id && !f.text.is_empty()
|
|
&& reply.as_ref().is_none_or(|r| *t <= r.0) => Some((*t, f.clone())),
|
|
_ => None,
|
|
}).collect();
|
|
let final_ev = finals.last().map(|(t, f)| (*t, HeardText {
|
|
text: finals.iter().map(|(_, f)| f.text.as_str()).collect::<Vec<_>>().join(" | "),
|
|
stt_ms: f.stt_ms,
|
|
}));
|
|
// The first sound is the earcon; the answer's own sound starts after its SpeakStart.
|
|
let reply_id = ev.iter().find_map(|(_, e)| match e {
|
|
CoreEvent::ReplyText { id, .. } => Some(id.clone()),
|
|
_ => None,
|
|
});
|
|
let speak_start = ev.iter().find_map(|(t, e)| match e {
|
|
CoreEvent::SpeakStart(id) if Some(id) == reply_id.as_ref() => Some(*t),
|
|
_ => None,
|
|
});
|
|
let audible = ears.audible.lock().unwrap().clone();
|
|
let first_sound = audible.first().copied();
|
|
let answer_sound = speak_start.and_then(|s| audible.iter().copied().find(|t| *t >= s));
|
|
rows.push(json!({
|
|
"round": round, "question": text,
|
|
"heard": final_ev.as_ref().map(|(_, f)| f.text.clone()),
|
|
"stt_ms": final_ev.as_ref().map(|(_, f)| f.stt_ms),
|
|
"answer": reply.as_ref().map(|r| r.1.clone()),
|
|
"final_ms": ms(final_ev.as_ref().map(|f| f.0)),
|
|
"answer_text_ms": ms(reply.as_ref().map(|r| r.0)),
|
|
"speak_start_ms": ms(speak_start),
|
|
"first_sound_ms": ms(first_sound),
|
|
"answer_sound_ms": ms(answer_sound),
|
|
"spoken": finished,
|
|
}));
|
|
println!("{phase} {}", rows.last().unwrap());
|
|
tokio::time::sleep(Duration::from_millis(1500)).await;
|
|
}
|
|
}
|
|
stop.store(true, Ordering::Relaxed);
|
|
let load_finals = match load { Some(h) => Some(h.await.unwrap()), None => None };
|
|
let stats = |key: &str| {
|
|
let mut v: Vec<i64> = rows.iter().filter_map(|r| r[key].as_i64()).collect();
|
|
v.sort_unstable();
|
|
let pick = |q: f64| v.get(((v.len() as f64 - 1.0) * q).round() as usize).copied();
|
|
(v.len(), json!({"min": v.first(), "median": pick(0.5), "p90": pick(0.9), "max": v.last()}))
|
|
};
|
|
let (answered, answer_sound) = stats("answer_sound_ms");
|
|
report.insert(phase.into(), json!({
|
|
"rows": rows, "answered": answered, "asked": rows.len(),
|
|
"first_sound_ms": stats("first_sound_ms").1,
|
|
"answer_sound_ms": answer_sound,
|
|
"background_finals": load_finals,
|
|
}));
|
|
}
|
|
println!("live round trip {}", serde_json::to_string_pretty(&Value::Object(report.clone())).unwrap());
|
|
})).await;
|
|
|
|
let _ = speaker_songbird.remove(GuildId::new(guild)).await;
|
|
s_shards.shutdown_all().await;
|
|
services.core.leave("test").await;
|
|
services.core.close().await;
|
|
services.worker.close();
|
|
services.worker_process.stop().await;
|
|
if let Err(panic) = steps {
|
|
std::panic::resume_unwind(panic);
|
|
}
|
|
// Every question is answered aloud.
|
|
for phase in ["alone", "with_9_more_voices"] {
|
|
let p = &report[phase];
|
|
assert_eq!(p["answered"], p["asked"], "{phase}: not every question got a spoken answer: {p}");
|
|
}
|
|
}
|
|
|