diff --git a/app/src/admin.spec.md b/app/src/admin.spec.md index e62d9d5..0030302 100644 --- a/app/src/admin.spec.md +++ b/app/src/admin.spec.md @@ -94,7 +94,10 @@ The dashboard's start and stop buttons, and the app binary itself, run the servi `Controller` for the data folder: states `stopped`, `starting`, `running`, `stopping`, `failed` (with the error, secrets masked). `start` from stopped or failed only, `stop` from running only; `restart` is stop then start. A start that fails leaves nothing running. Starting reads the -settings and the stored token again, so saved changes take effect on the next start. +settings and the stored token again, so saved changes take effect on the next start. Each start +gets a new dashboard fed by that run's events; `stop` ends the feeding (and waits for it) before +the dashboard is forgotten, so an event queued before the stop, or sent by an earlier run, never +reaches the screen. ## Dashboard data (`app/src/dashboard.rs`) @@ -105,12 +108,14 @@ settings and the stored token again, so saved changes take effect on the next st - `voice`: the voice channel Jarvis is in and the people there (ID and display name), as the core reports them to the worker. - `captions`: live captions, the last 30 lines: a speaker's partial transcript is replaced by - their next partial and then by the final. On the screen only: never logged, never written, + their next partial and then by the final. Only while Jarvis is in a voice channel: a transcript + that arrives after it left is not shown. On the screen only: never logged, never written, cleared when Jarvis leaves the voice channel or the services stop. - `latency`: the last 20 voice answers, each split into its stages: speech end to final transcript, the brain (answer asked to answer text), the voice (answer text to the first sound) and the - total; with the median of each. An answer is matched to its question by the core's - `AnswerAsked { id, user }` event (the speech id and the asker) and that user's latest final. + total; with the median of each. The core's `AnswerAsked { id, user, ended_at, final_at }` event + carries the speech id and the request's own times (when its speech ended, when its final + arrived), so a second question waiting behind the first does not change the first's times. - `missed`: the conversation's missed requests (`conversation.spec.md`), newest first, at most 20: who, why and how long ago, never the text. - `system`: memory in use and total, and per NVIDIA GPU its use and VRAM (from `nvidia-smi`, diff --git a/app/src/control.rs b/app/src/control.rs index 9358079..4c9dcfb 100644 --- a/app/src/control.rs +++ b/app/src/control.rs @@ -43,19 +43,22 @@ struct Inner { state: State, error: Option, running: Option>, + /// The running services' dashboard and the task feeding it their events. Each start gets a new + /// one, so an event of an earlier run can never reach a later dashboard. + dashboard: Arc>, + feeder: Option>, } pub struct Controller { host: Box, inner: Mutex, - dashboard: Arc>, usage: tokio::sync::Mutex>, } impl Controller { pub fn new(host: impl Host) -> Arc { - Arc::new(Self { host: Box::new(host), inner: Mutex::new(Inner { state: State::Stopped, error: None, running: None }), - dashboard: Arc::default(), usage: tokio::sync::Mutex::new(None) }) + Arc::new(Self { host: Box::new(host), inner: Mutex::new(Inner { state: State::Stopped, error: None, running: None, + dashboard: Arc::default(), feeder: None }), usage: tokio::sync::Mutex::new(None) }) } pub fn state(&self) -> State { @@ -75,15 +78,18 @@ impl Controller { crate::info_log!("app", "starting the services"); match self.host.start().await { Ok((running, mut events)) => { - let dash = self.dashboard.clone(); - tokio::spawn(async move { + let dash: Arc> = Arc::default(); + let d2 = dash.clone(); + let feeder = tokio::spawn(async move { while let Some(ev) = events.recv().await { - dash.lock().unwrap().on_event(&ev, Instant::now()); + d2.lock().unwrap().on_event(&ev, Instant::now()); } }); let mut i = self.inner.lock().unwrap(); i.state = State::Running; i.running = Some(running); + i.dashboard = dash; + i.feeder = Some(feeder); Ok(()) } Err(e) => { @@ -110,8 +116,16 @@ impl Controller { if let Some(r) = running { r.stop().await; } - self.dashboard.lock().unwrap().clear(); - self.inner.lock().unwrap().state = State::Stopped; + // Stop feeding before forgetting: nothing still queued may reach the screen afterwards. + let feeder = self.inner.lock().unwrap().feeder.take(); + if let Some(f) = feeder { + f.abort(); + let _ = f.await; + } + let mut i = self.inner.lock().unwrap(); + i.dashboard = Arc::default(); + i.state = State::Stopped; + drop(i); crate::info_log!("app", "services stopped"); Ok(()) } @@ -126,13 +140,13 @@ impl Controller { /// Everything the dashboard shows. pub async fn snapshot(&self) -> Value { let now = Instant::now(); - let (state, error, running) = { + let (state, error, running, dashboard) = { let i = self.inner.lock().unwrap(); - (i.state, i.error.clone(), i.running.clone()) + (i.state, i.error.clone(), i.running.clone(), i.dashboard.clone()) }; let status = running.map(|r| r.status(now)).unwrap_or_else(|| json!({"worker": null, "voiceMembers": [], "missed": []})); - let mut view = self.dashboard.lock().unwrap().view(&[], now); + let mut view = dashboard.lock().unwrap().view(&[], now); view["missed"] = status["missed"].clone(); view["worker"] = status["worker"].clone(); view["voiceMembers"] = status["voiceMembers"].clone(); @@ -294,6 +308,27 @@ mod tests { assert_eq!(counts.stops.load(Ordering::SeqCst), 0, "a failed start leaves nothing to stop"); } + #[tokio::test] + async fn events_queued_before_a_stop_or_sent_by_an_earlier_run_never_reach_the_screen() { + let (c, _, host) = controller(0); + c.start().await.unwrap(); + let old = host.events.lock().unwrap().clone().unwrap(); + // Queued right before the stop, never handled before it. + old.send(CoreEvent::Joined { guild_id: 1, channel_id: 77 }).unwrap(); + old.send(CoreEvent::Partial(crate::discord::Heard { speaker_id: 11, speaker_name: "민수".into(), utt: 1 }, + "자비스".into())).unwrap(); + c.stop().await.unwrap(); + tokio::time::sleep(Duration::from_millis(30)).await; + let s = c.snapshot().await; + assert_eq!((s["state"].clone(), s["voiceChannel"].clone(), s["captions"].clone()), + (json!("stopped"), Value::Null, json!([]))); + // After a new start, the earlier run's sender still reaches nothing. + c.start().await.unwrap(); + let _ = old.send(CoreEvent::Joined { guild_id: 1, channel_id: 78 }); + tokio::time::sleep(Duration::from_millis(30)).await; + assert_eq!(c.snapshot().await["voiceChannel"], Value::Null); + } + #[tokio::test] async fn the_snapshot_carries_the_events_and_stopping_forgets_what_was_said() { let (c, _, host) = controller(0); diff --git a/app/src/conversation.rs b/app/src/conversation.rs index 8b5571b..82cb640 100644 --- a/app/src/conversation.rs +++ b/app/src/conversation.rs @@ -22,8 +22,9 @@ pub const MISSED_KEPT: usize = 100; /// What the conversation needs from the voice side. pub trait Voice { - /// Have the brain answer and speak it; the speech request id, or None when it cannot. - fn answer(&self, user: u64, text: &str) -> Option; + /// Have the brain answer and speak it; the speech request id, or None when it cannot. `heard`: + /// when the request's speech ended and its final arrived (the dashboard times the answer by it). + fn answer(&self, user: u64, text: &str, heard: (Instant, Instant)) -> Option; /// Stop one speech request. fn cancel(&self, id: &str); /// Stop all speech at once. @@ -44,8 +45,8 @@ pub trait Voice { } impl Voice for DiscordCore { - fn answer(&self, user: u64, text: &str) -> Option { - DiscordCore::answer(self, user, text) + fn answer(&self, user: u64, text: &str, heard: (Instant, Instant)) -> Option { + DiscordCore::answer(self, user, text, heard) } fn cancel(&self, id: &str) { DiscordCore::cancel(self, id) @@ -402,7 +403,7 @@ impl Conversation { if self.calls.earcon && !self.chimed.remove(&r.id) { self.voice.earcon(); } - self.voice.answer(r.speaker, &r.text) + self.voice.answer(r.speaker, &r.text, (r.ended_at, r.arrived_at)) } { Some(sid) => { debug_log!("conversation", "answer req={} speaker={} speech={sid}", r.id, r.speaker); @@ -494,7 +495,7 @@ mod tests { n: Rc>, } impl Voice for Fake { - fn answer(&self, user: u64, text: &str) -> Option { + fn answer(&self, user: u64, text: &str, _heard: (Instant, Instant)) -> Option { *self.n.borrow_mut() += 1; self.did.borrow_mut().push(Did::Answer(user, text.into())); Some(format!("s{}", self.n.borrow())) diff --git a/app/src/dashboard.rs b/app/src/dashboard.rs index 46c83db..f5a1db2 100644 --- a/app/src/dashboard.rs +++ b/app/src/dashboard.rs @@ -4,7 +4,7 @@ //! people said for the screen only; nothing here is logged or written. use std::collections::{HashMap, VecDeque}; -use std::time::{Duration, Instant}; +use std::time::Instant; use serde::Serialize; use serde_json::{json, Value}; @@ -27,7 +27,8 @@ pub struct Caption { utt: u32, } -/// One voice answer's stages in milliseconds (None: that stage could not be seen). +/// One voice answer's stages in milliseconds (None: that stage could not be seen, e.g. no answer text +/// before the sound). #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Default)] #[serde(rename_all = "camelCase")] pub struct AnswerTiming { @@ -43,8 +44,8 @@ pub struct AnswerTiming { struct Pending { asked: Instant, - /// The asker's latest final: when their speech ended and when the final came. - heard: Option<(Instant, Instant)>, + /// When the request's speech ended and when its final came. + heard: (Instant, Instant), reply: Option, } @@ -53,7 +54,6 @@ pub struct Dashboard { bot_tag: Option, voice_channel: Option, captions: VecDeque, - last_final: HashMap, pending: HashMap, answers: VecDeque, } @@ -72,18 +72,16 @@ impl Dashboard { self.captions.clear(); self.pending.clear(); } - CoreEvent::Partial(h, text) => self.caption(h.speaker_id, &h.speaker_name, h.utt, text, false), - CoreEvent::Final(f) => { + // Captions belong to the voice session: a transcript arriving after Jarvis left is dropped. + CoreEvent::Partial(h, text) if self.voice_channel.is_some() => + self.caption(h.speaker_id, &h.speaker_name, h.utt, text, false), + CoreEvent::Final(f) if self.voice_channel.is_some() => { let h = &f.heard; self.caption(h.speaker_id, &h.speaker_name, h.utt, &f.text, true); - if !f.text.trim().is_empty() { - let ended = now.checked_sub(Duration::from_millis(f.wait_ms)).unwrap_or(now); - self.last_final.insert(h.speaker_id, (ended, now)); - } } - CoreEvent::AnswerAsked { id, user } => { - let heard = self.last_final.get(user).copied(); - self.pending.insert(id.clone(), Pending { asked: now, heard, reply: None }); + // The request's own times: a second question waiting behind the first does not move them. + CoreEvent::AnswerAsked { id, ended_at, final_at, .. } => { + self.pending.insert(id.clone(), Pending { asked: now, heard: (*ended_at, *final_at), reply: None }); } CoreEvent::ReplyText { id, .. } => { if let Some(p) = self.pending.get_mut(id) { @@ -93,10 +91,10 @@ impl Dashboard { CoreEvent::SpeakStart(id) => { if let Some(p) = self.pending.remove(id) { let t = AnswerTiming { - final_ms: p.heard.map(|(e, f)| ms(e, f)), + final_ms: Some(ms(p.heard.0, p.heard.1)), brain_ms: p.reply.map(|r| ms(p.asked, r)), voice_ms: p.reply.map(|r| ms(r, now)), - total_ms: p.heard.map(|(e, _)| ms(e, now)), + total_ms: Some(ms(p.heard.0, now)), }; if self.answers.len() == ANSWERS_KEPT { self.answers.pop_front(); @@ -180,6 +178,7 @@ mod tests { use crate::discord::{Heard, HeardFinal}; use crate::playback::PlaybackEnd; use crate::policy::MissReason; + use std::time::Duration; const A: u64 = 11; fn heard(utt: u32) -> Heard { @@ -192,10 +191,23 @@ mod tests { fn at(t: Instant, ms: u64) -> Instant { t + Duration::from_millis(ms) } + fn joined() -> CoreEvent { + CoreEvent::Joined { guild_id: 1, channel_id: 5 } + } + fn left() -> CoreEvent { + CoreEvent::Left { guild_id: 1, channel_id: 5, reason: "command".into() } + } + fn asked(id: &str, t: Instant, ended: u64, final_at: u64) -> CoreEvent { + CoreEvent::AnswerAsked { id: id.into(), user: A, ended_at: at(t, ended), final_at: at(t, final_at) } + } + fn reply(id: &str) -> CoreEvent { + CoreEvent::ReplyText { id: id.into(), text: "네".into(), failed: false } + } #[test] fn a_partial_is_replaced_by_the_next_and_then_by_the_final() { let (mut d, t) = (Dashboard::default(), Instant::now()); + d.on_event(&joined(), t); d.on_event(&CoreEvent::Partial(heard(1), "자비스 오늘".into()), t); d.on_event(&CoreEvent::Partial(heard(1), "자비스 오늘 날씨".into()), t); assert_eq!(d.captions().iter().map(|c| (c.text.as_str(), c.is_final)).collect::>(), @@ -211,31 +223,35 @@ mod tests { } #[test] - fn captions_are_capped_and_cleared_when_jarvis_leaves() { + fn captions_are_capped_and_cleared_when_jarvis_leaves_and_late_ones_stay_out() { let (mut d, t) = (Dashboard::default(), Instant::now()); - d.on_event(&CoreEvent::Joined { guild_id: 1, channel_id: 5 }, t); + d.on_event(&joined(), t); for i in 0..(CAPTIONS_KEPT as u32 + 5) { d.on_event(&fin(i, &format!("말 {i}"), 100), t); } let caps = d.captions(); assert_eq!(caps.len(), CAPTIONS_KEPT); assert_eq!(caps.last().unwrap().text, format!("말 {}", CAPTIONS_KEPT + 4)); - d.on_event(&CoreEvent::Left { guild_id: 1, channel_id: 5, reason: "command".into() }, t); + d.on_event(&left(), t); + assert!(d.captions().is_empty()); + // A transcript of the session that ended arrives after the leave: it does not come back. + d.on_event(&CoreEvent::Partial(heard(90), "늦게 온 말".into()), t); + d.on_event(&fin(90, "늦게 온 말", 100), t); assert!(d.captions().is_empty()); assert_eq!(d.view(&[], t)["voiceChannel"], Value::Null); } #[test] - fn an_answer_is_timed_from_its_askers_speech_end_to_its_first_sound() { + fn an_answer_is_timed_from_its_own_requests_speech_end_to_its_first_sound() { let (mut d, t) = (Dashboard::default(), Instant::now()); - d.on_event(&fin(1, "자비스 몇 시야", 700), at(t, 1000)); // speech ended at 300 - d.on_event(&CoreEvent::AnswerAsked { id: "r1".into(), user: A }, at(t, 1010)); - d.on_event(&CoreEvent::ReplyText { id: "r1".into(), text: "열 시입니다.".into(), failed: false }, at(t, 1700)); + d.on_event(&joined(), t); + d.on_event(&asked("r1", t, 300, 1000), at(t, 1010)); + d.on_event(&reply("r1"), at(t, 1700)); d.on_event(&CoreEvent::SpeakStart("r1".into()), at(t, 1900)); assert_eq!(d.answers(), vec![AnswerTiming { final_ms: Some(700), brain_ms: Some(690), voice_ms: Some(200), total_ms: Some(1600) }]); // An answer stopped before it started is not timed; the earcon (another id) is not either. - d.on_event(&CoreEvent::AnswerAsked { id: "r3".into(), user: A }, at(t, 3000)); + d.on_event(&asked("r3", t, 2000, 2500), at(t, 3000)); d.on_event(&CoreEvent::SpeakStart("r2".into()), at(t, 3000)); d.on_event(&CoreEvent::SpeakEnd(PlaybackEnd { id: "r3".into(), played: 0, cancelled: true, failed: false }), at(t, 3100)); @@ -243,25 +259,38 @@ mod tests { assert_eq!(d.answers().len(), 1); } + #[test] + fn a_second_question_waiting_does_not_change_the_first_answers_times() { + let (mut d, t) = (Dashboard::default(), Instant::now()); + d.on_event(&joined(), t); + // First question ends at 0 (final at 700); the same person asks again, ending at 1000, while + // the first is still answered. The first answer starts sounding at 3700. + d.on_event(&fin(1, "첫 질문", 700), at(t, 700)); + d.on_event(&asked("r1", t, 0, 700), at(t, 710)); + d.on_event(&fin(2, "둘째 질문", 700), at(t, 1700)); + d.on_event(&reply("r1"), at(t, 3500)); + d.on_event(&CoreEvent::SpeakStart("r1".into()), at(t, 3700)); + assert_eq!(d.answers()[0].total_ms, Some(3700)); + assert_eq!(d.answers()[0].final_ms, Some(700)); + } + #[test] fn the_medians_cover_the_last_answers_kept() { let (mut d, t) = (Dashboard::default(), Instant::now()); for (i, brain) in [500u64, 900, 700].into_iter().enumerate() { let base = i as u64 * 10_000; - d.on_event(&fin(i as u32, "자비스", 600), at(t, base + 1000)); - d.on_event(&CoreEvent::AnswerAsked { id: format!("r{i}"), user: A }, at(t, base + 1000)); - d.on_event(&CoreEvent::ReplyText { id: format!("r{i}"), text: "네".into(), failed: false }, - at(t, base + 1000 + brain)); + d.on_event(&asked(&format!("r{i}"), t, base + 400, base + 1000), at(t, base + 1000)); + d.on_event(&reply(&format!("r{i}")), at(t, base + 1000 + brain)); d.on_event(&CoreEvent::SpeakStart(format!("r{i}")), at(t, base + 1000 + brain + 100)); } let m = d.medians(); assert_eq!((m.final_ms, m.brain_ms, m.voice_ms, m.total_ms), (Some(600), Some(700), Some(100), Some(1400))); for i in 0..ANSWERS_KEPT as u64 { - d.on_event(&CoreEvent::AnswerAsked { id: format!("x{i}"), user: 99 }, at(t, 100_000 + i)); + d.on_event(&asked(&format!("x{i}"), t, 100_000, 100_000), at(t, 100_000 + i)); d.on_event(&CoreEvent::SpeakStart(format!("x{i}")), at(t, 100_000 + i)); } assert_eq!(d.answers().len(), ANSWERS_KEPT); - assert_eq!(d.medians().final_ms, None, "no final seen for the asker of the answers kept"); + assert_eq!(d.medians().brain_ms, None, "no answer text seen for the answers kept"); } #[test] diff --git a/app/src/discord.rs b/app/src/discord.rs index 04d7ef7..65e8c9c 100644 --- a/app/src/discord.rs +++ b/app/src/discord.rs @@ -76,8 +76,9 @@ pub enum CoreEvent { /// The voice worker connection ended: finals of utterances in progress will not come, and the /// restarted worker numbers utterances afresh. WorkerLost, - /// `answer(user, ..)` queued speech `id` (the dashboard times answers by it). - AnswerAsked { id: String, user: u64 }, + /// `answer(user, ..)` queued speech `id` for a request whose speech ended at `ended_at` and whose + /// final arrived at `final_at` (the dashboard times answers by it). + AnswerAsked { id: String, user: u64, ended_at: Instant, final_at: Instant }, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -424,11 +425,13 @@ impl DiscordCore { /// Have the brain answer `user` and speak it as one request in the queue. The request id (its /// `SpeakStart` and `SpeakEnd` follow as for `say`), or None when not in a voice channel. It ends as - /// failed when the brain gives no answer. - pub fn answer(&self, user: u64, text: &str) -> Option { + /// failed when the brain gives no answer. `heard`: when the request's speech ended and its final + /// arrived, passed on in `AnswerAsked`. + pub fn answer(&self, user: u64, text: &str, heard: (Instant, Instant)) -> Option { self.voice_channel_id()?; let id = with_playback(&self.inner, |p| p.adopt()); - let _ = self.inner.events.send(CoreEvent::AnswerAsked { id: id.clone(), user }); + let _ = self.inner.events.send(CoreEvent::AnswerAsked { id: id.clone(), user, ended_at: heard.0, + final_at: heard.1 }); self.inner.worker.reply(&id, user, text, "voice", true); Some(id) } diff --git a/app/src/discord_core.spec.md b/app/src/discord_core.spec.md index fce770a..77cf3d8 100644 --- a/app/src/discord_core.spec.md +++ b/app/src/discord_core.spec.md @@ -145,8 +145,9 @@ and the late "left" update cancelled the new join (seen on real Discord). `SpeakStart` and `SpeakEnd`; `Stopped` when all speech is stopped (`stop()`, the slash command or the session ending), sent before the `SpeakEnd`s it causes; `Joined` and `Left` (with a reason: `command`, `moved`, `disconnected`, `shutdown`); `Addressed` and `Ranked` with the brain's answers to `ask_addressed` and `ask_rank`. `WorkerLost` when the voice worker connection ends: the finals of utterances in progress will not - come, and the restarted worker numbers utterances afresh. `AnswerAsked { id, user }` when - `answer(user, ..)` queued speech `id` (the dashboard times answers by it, `admin.spec.md`). + come, and the restarted worker numbers utterances afresh. `AnswerAsked { id, user, ended_at, + final_at }` when `answer(user, text, heard)` queued speech `id` for a request whose speech ended + and whose final arrived at those times (the dashboard times answers by it, `admin.spec.md`). `voice_members()` lists the people in Jarvis's voice channel that it hears (ID and display name), the same set reported to the worker. - **Speaking**: `say(text, more)` queues a TTS request and returns its id (`None` when not in a voice