diff --git a/app/src/admin.spec.md b/app/src/admin.spec.md index 0030302..d4d3c1e 100644 --- a/app/src/admin.spec.md +++ b/app/src/admin.spec.md @@ -108,8 +108,9 @@ reaches the screen. - `voice`: the voice channel Jarvis is in and the people there (ID and display name), as the core reports them to the worker. - `captions`: live captions, the last 30 lines: a speaker's partial transcript is replaced by - their next partial and then by the final. Only while Jarvis is in a voice channel: a transcript - that arrives after it left is not shown. On the screen only: never logged, never written, + their next partial and then by the final. A transcript belongs to the voice session its + utterance started in (`Heard.session`, `discord_core.spec.md`): only the current session's are + shown, so one arriving after Jarvis left, or after it moved to another channel, is not. On the screen only: never logged, never written, cleared when Jarvis leaves the voice channel or the services stop. - `latency`: the last 20 voice answers, each split into its stages: speech end to final transcript, the brain (answer asked to answer text), the voice (answer text to the first sound) and the diff --git a/app/src/control.rs b/app/src/control.rs index 4c9dcfb..a394ddb 100644 --- a/app/src/control.rs +++ b/app/src/control.rs @@ -314,8 +314,8 @@ mod tests { c.start().await.unwrap(); let old = host.events.lock().unwrap().clone().unwrap(); // Queued right before the stop, never handled before it. - old.send(CoreEvent::Joined { guild_id: 1, channel_id: 77 }).unwrap(); - old.send(CoreEvent::Partial(crate::discord::Heard { speaker_id: 11, speaker_name: "민수".into(), utt: 1 }, + old.send(CoreEvent::Joined { guild_id: 1, channel_id: 77, session: 1 }).unwrap(); + old.send(CoreEvent::Partial(crate::discord::Heard { speaker_id: 11, speaker_name: "민수".into(), utt: 1, session: Some(1) }, "자비스".into())).unwrap(); c.stop().await.unwrap(); tokio::time::sleep(Duration::from_millis(30)).await; @@ -324,7 +324,7 @@ mod tests { (json!("stopped"), Value::Null, json!([]))); // After a new start, the earlier run's sender still reaches nothing. c.start().await.unwrap(); - let _ = old.send(CoreEvent::Joined { guild_id: 1, channel_id: 78 }); + let _ = old.send(CoreEvent::Joined { guild_id: 1, channel_id: 78, session: 2 }); tokio::time::sleep(Duration::from_millis(30)).await; assert_eq!(c.snapshot().await["voiceChannel"], Value::Null); } @@ -335,8 +335,8 @@ mod tests { c.start().await.unwrap(); let tx = host.events.lock().unwrap().clone().unwrap(); tx.send(CoreEvent::Ready { user_id: 1, tag: "Jarvis#0001".into() }).unwrap(); - tx.send(CoreEvent::Joined { guild_id: 1, channel_id: 77 }).unwrap(); - tx.send(CoreEvent::Partial(crate::discord::Heard { speaker_id: 11, speaker_name: "민수".into(), utt: 1 }, + tx.send(CoreEvent::Joined { guild_id: 1, channel_id: 77, session: 1 }).unwrap(); + tx.send(CoreEvent::Partial(crate::discord::Heard { speaker_id: 11, speaker_name: "민수".into(), utt: 1, session: Some(1) }, "자비스 안녕".into())).unwrap(); tokio::time::sleep(Duration::from_millis(50)).await; let s = c.snapshot().await; diff --git a/app/src/conversation.rs b/app/src/conversation.rs index 82cb640..1093f20 100644 --- a/app/src/conversation.rs +++ b/app/src/conversation.rs @@ -592,7 +592,7 @@ mod tests { } fn partial(speaker: u64, text: &str) -> CoreEvent { - CoreEvent::Partial(Heard { speaker_id: speaker, speaker_name: "x".into(), utt: 1 }, text.into()) + CoreEvent::Partial(Heard { speaker_id: speaker, speaker_name: "x".into(), utt: 1, session: Some(1) }, text.into()) } #[test] @@ -622,7 +622,7 @@ mod tests { assert_eq!(f.take(), vec![Did::Answer(A, "자비스 안녕".into())]); } fn fin(speaker: u64, text: &str, wait_ms: u64) -> CoreEvent { - CoreEvent::Final(HeardFinal { heard: Heard { speaker_id: speaker, speaker_name: "x".into(), utt: 1 }, + CoreEvent::Final(HeardFinal { heard: Heard { speaker_id: speaker, speaker_name: "x".into(), utt: 1, session: Some(1) }, text: text.into(), end_reason: "silence".into(), audio_ms: 1000, wait_ms, stt_ms: 100 }) } fn end(id: &str, cancelled: bool) -> CoreEvent { @@ -633,11 +633,11 @@ mod tests { } fn fin_utt(speaker: u64, utt: u32, text: &str) -> CoreEvent { - CoreEvent::Final(HeardFinal { heard: Heard { speaker_id: speaker, speaker_name: "x".into(), utt }, + CoreEvent::Final(HeardFinal { heard: Heard { speaker_id: speaker, speaker_name: "x".into(), utt, session: Some(1) }, text: text.into(), end_reason: "silence".into(), audio_ms: 1000, wait_ms: 100, stt_ms: 100 }) } fn start(speaker: u64, utt: u32) -> CoreEvent { - CoreEvent::SpeechStart(Heard { speaker_id: speaker, speaker_name: "x".into(), utt }) + CoreEvent::SpeechStart(Heard { speaker_id: speaker, speaker_name: "x".into(), utt, session: Some(1) }) } #[test] diff --git a/app/src/dashboard.rs b/app/src/dashboard.rs index f5a1db2..6b80462 100644 --- a/app/src/dashboard.rs +++ b/app/src/dashboard.rs @@ -53,6 +53,8 @@ struct Pending { pub struct Dashboard { bot_tag: Option, voice_channel: Option, + /// The voice session Jarvis is in (from `Joined`). + session: Option, captions: VecDeque, pending: HashMap, answers: VecDeque, @@ -66,16 +68,21 @@ impl Dashboard { pub fn on_event(&mut self, ev: &CoreEvent, now: Instant) { match ev { CoreEvent::Ready { tag, .. } => self.bot_tag = Some(tag.clone()), - CoreEvent::Joined { channel_id, .. } => self.voice_channel = Some(*channel_id), + CoreEvent::Joined { channel_id, session, .. } => { + self.voice_channel = Some(*channel_id); + self.session = Some(*session); + } CoreEvent::Left { .. } => { self.voice_channel = None; + self.session = None; self.captions.clear(); self.pending.clear(); } - // Captions belong to the voice session: a transcript arriving after Jarvis left is dropped. - CoreEvent::Partial(h, text) if self.voice_channel.is_some() => + // A transcript belongs to the session its utterance started in: one of an earlier session + // (arriving after a leave, or after a move to another channel) is not shown. + CoreEvent::Partial(h, text) if self.session.is_some() && h.session == self.session => self.caption(h.speaker_id, &h.speaker_name, h.utt, text, false), - CoreEvent::Final(f) if self.voice_channel.is_some() => { + CoreEvent::Final(f) if self.session.is_some() && f.heard.session == self.session => { let h = &f.heard; self.caption(h.speaker_id, &h.speaker_name, h.utt, &f.text, true); } @@ -181,18 +188,24 @@ mod tests { use std::time::Duration; const A: u64 = 11; + fn heard_in(utt: u32, session: u64) -> Heard { + Heard { speaker_id: A, speaker_name: "민수".into(), utt, session: Some(session) } + } fn heard(utt: u32) -> Heard { - Heard { speaker_id: A, speaker_name: "민수".into(), utt } + heard_in(utt, 1) + } + fn fin_in(utt: u32, text: &str, wait_ms: u64, session: u64) -> CoreEvent { + CoreEvent::Final(HeardFinal { heard: heard_in(utt, session), text: text.into(), end_reason: "silence".into(), + audio_ms: 900, wait_ms, stt_ms: 200 }) } fn fin(utt: u32, text: &str, wait_ms: u64) -> CoreEvent { - CoreEvent::Final(HeardFinal { heard: heard(utt), text: text.into(), end_reason: "silence".into(), - audio_ms: 900, wait_ms, stt_ms: 200 }) + fin_in(utt, text, wait_ms, 1) } fn at(t: Instant, ms: u64) -> Instant { t + Duration::from_millis(ms) } fn joined() -> CoreEvent { - CoreEvent::Joined { guild_id: 1, channel_id: 5 } + CoreEvent::Joined { guild_id: 1, channel_id: 5, session: 1 } } fn left() -> CoreEvent { CoreEvent::Left { guild_id: 1, channel_id: 5, reason: "command".into() } @@ -241,6 +254,25 @@ mod tests { assert_eq!(d.view(&[], t)["voiceChannel"], Value::Null); } + #[test] + fn a_transcript_of_the_channel_jarvis_moved_away_from_stays_out_of_the_new_ones_captions() { + let (mut d, t) = (Dashboard::default(), Instant::now()); + d.on_event(&joined(), t); + d.on_event(&CoreEvent::SpeechStart(heard_in(7, 1)), t); // started in channel A, session 1 + d.on_event(&left(), t); + d.on_event(&CoreEvent::Joined { guild_id: 1, channel_id: 6, session: 2 }, t); + d.on_event(&CoreEvent::Partial(heard_in(7, 1), "A에서 한 말".into()), t); + d.on_event(&fin_in(7, "A에서 한 말", 300, 1), t); + assert!(d.captions().is_empty(), "session 1's utterance is not session 2's"); + // An utterance whose start was not seen in a session (None) is not shown either. + d.on_event(&CoreEvent::Partial(Heard { speaker_id: A, speaker_name: "민수".into(), utt: 8, session: None }, + "어디서 온 말".into()), t); + assert!(d.captions().is_empty()); + // The same person speaking in B is. + d.on_event(&fin_in(9, "B에서 한 말", 300, 2), t); + assert_eq!(d.captions().iter().map(|c| c.text.as_str()).collect::>(), vec!["B에서 한 말"]); + } + #[test] fn an_answer_is_timed_from_its_own_requests_speech_end_to_its_first_sound() { let (mut d, t) = (Dashboard::default(), Instant::now()); diff --git a/app/src/discord.rs b/app/src/discord.rs index 65e8c9c..970ad43 100644 --- a/app/src/discord.rs +++ b/app/src/discord.rs @@ -39,6 +39,10 @@ pub struct Heard { pub speaker_id: u64, pub speaker_name: String, pub utt: u32, + /// The voice session the utterance started in (the number its `Joined` carried); None when its + /// start was not seen in a session. A transcript belongs to the session its utterance started + /// in, even when it arrives after Jarvis left or moved on. + pub session: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -54,7 +58,8 @@ pub struct HeardFinal { #[derive(Debug, Clone, PartialEq, Eq)] pub enum CoreEvent { Ready { user_id: u64, tag: String }, - Joined { guild_id: u64, channel_id: u64 }, + /// Jarvis joined a voice channel; `session` numbers each voice session from 1. + Joined { guild_id: u64, channel_id: u64, session: u64 }, Left { guild_id: u64, channel_id: u64, reason: String }, SpeechStart(Heard), /// The speaker stopped sending audio (their final is still to come). @@ -191,6 +196,34 @@ struct Inner { /// Serialises join and leave. voice_lock: tokio::sync::Mutex<()>, names: Mutex>, + /// The latest voice session's number, and the session each utterance in progress started in. + voice_session: std::sync::atomic::AtomicU64, + utt_sessions: Mutex, +} + +/// The voice session each utterance in progress started in (`Heard::session`). +#[derive(Default)] +pub struct UttSessions(HashMap<(u64, u32), u64>); + +impl UttSessions { + /// Utterance `(speaker, utt)` started; `current` is the session Jarvis is in (None: none). + pub fn started(&mut self, speaker: u64, utt: u32, current: Option) { + match current { + Some(s) => self.0.insert((speaker, utt), s), + None => self.0.remove(&(speaker, utt)), + }; + } + pub fn session_of(&self, speaker: u64, utt: u32) -> Option { + self.0.get(&(speaker, utt)).copied() + } + /// Its final came: nothing more of it will. + pub fn finished(&mut self, speaker: u64, utt: u32) { + self.0.remove(&(speaker, utt)); + } + /// The session ended or the worker restarted: no utterance in progress belongs to a session now. + pub fn clear(&mut self) { + self.0.clear(); + } } /// Handle to the core; cheap to clone. @@ -225,6 +258,8 @@ impl DiscordCore { session: Mutex::new(None), voice_lock: tokio::sync::Mutex::new(()), names: Mutex::new(HashMap::new()), + voice_session: std::sync::atomic::AtomicU64::new(0), + utt_sessions: Mutex::new(UttSessions::default()), }); tokio::spawn(player_task(Arc::downgrade(&inner), player_rx)); tokio::spawn(worker_task(Arc::downgrade(&inner), worker_events)); @@ -391,11 +426,13 @@ impl DiscordCore { } let alive = Arc::new(AtomicBool::new(true)); tokio::spawn(speaking_end_timer(Arc::downgrade(&self.inner), rx.clone(), alive.clone())); + // A new session number before any of its utterances can start. + let session = self.inner.voice_session.fetch_add(1, Ordering::SeqCst) + 1; *self.inner.session.lock().unwrap() = Some(Session { guild_id: gc.guild_id.get(), channel_id, rx, alive }); let _ = self.inner.player.send(PlayerCmd::SetCall(Some(call))); report_voice_state(&self.inner); debug_log!("voice", "joined voice {channel_id}"); - let _ = self.inner.events.send(CoreEvent::Joined { guild_id: gc.guild_id.get(), channel_id }); + let _ = self.inner.events.send(CoreEvent::Joined { guild_id: gc.guild_id.get(), channel_id, session }); let notice = CreateMessage::new().content(&self.inner.settings.voice.join_notice) .allowed_mentions(CreateAllowedMentions::new()); if let Err(e) = gc.id.send_message(&http, notice).await { @@ -615,6 +652,8 @@ fn with_playback(inner: &Inner, f: impl FnOnce(&mut Playback, s: Session, reason: &str, disconnect: bool) { s.alive.store(false, Ordering::Relaxed); + // Utterances still in progress carry no session from here: their late transcripts belong to none. + inner.utt_sessions.lock().unwrap().clear(); let _ = inner.events.send(CoreEvent::Stopped); with_playback(inner, |p| p.stop()); let _ = inner.player.send(PlayerCmd::SetCall(None)); @@ -693,7 +732,15 @@ fn name_of(inner: &Inner, user: u64) -> String { fn heard(inner: &Inner, speaker: &str, utt: u32) -> Option { let id: u64 = speaker.parse().ok()?; - Some(Heard { speaker_id: id, speaker_name: name_of(inner, id), utt }) + let session = inner.utt_sessions.lock().unwrap().session_of(id, utt); + Some(Heard { speaker_id: id, speaker_name: name_of(inner, id), utt, session }) +} + +/// An utterance started: it belongs to the session Jarvis is in now (none outside one). +fn utterance_started(inner: &Inner, speaker: &str, utt: u32) { + let Ok(id) = speaker.parse::() else { return }; + let current = inner.session.lock().unwrap().is_some().then(|| inner.voice_session.load(Ordering::SeqCst)); + inner.utt_sessions.lock().unwrap().started(id, utt, current); } fn to_chat(inner: &Inner, m: &Message) -> ChatMessage { @@ -716,6 +763,7 @@ async fn worker_task(core: std::sync::Weak, mut rx: mpsc::UnboundedReceiv let Some(inner) = core.upgrade() else { return }; match ev { WorkerEvent::SpeechStart(m) => { + utterance_started(&inner, &m.speaker, m.utt); if let Some(h) = heard(&inner, &m.speaker, m.utt) { let _ = inner.events.send(CoreEvent::SpeechStart(h)); } @@ -753,6 +801,8 @@ async fn worker_task(core: std::sync::Weak, mut rx: mpsc::UnboundedReceiv }); } WorkerEvent::Disconnected => { + // The restarted worker numbers utterances afresh: the old ones' sessions are void. + inner.utt_sessions.lock().unwrap().clear(); let _ = inner.events.send(CoreEvent::WorkerLost); } WorkerEvent::Error(_) => {} @@ -763,7 +813,11 @@ async fn worker_task(core: std::sync::Weak, mut rx: mpsc::UnboundedReceiv fn on_final(inner: &Inner, m: Final) { debug_log!("voice", "final speaker={} utt={} reason={} chars={} audio_ms={} wait_ms={}", m.speaker, m.utt, m.end_reason, m.text.chars().count(), m.audio_ms, m.wait_ms); - if let Some(h) = heard(inner, &m.speaker, m.utt) { + let h = heard(inner, &m.speaker, m.utt); + if let Ok(id) = m.speaker.parse::() { + inner.utt_sessions.lock().unwrap().finished(id, m.utt); + } + if let Some(h) = h { let _ = inner.events.send(CoreEvent::Final(HeardFinal { heard: h, text: m.text, end_reason: m.end_reason, audio_ms: m.audio_ms, wait_ms: m.wait_ms, stt_ms: m.stt_ms })); } @@ -1090,6 +1144,20 @@ impl Handler { #[cfg(test)] mod tests { + #[test] + fn an_utterance_keeps_the_session_it_started_in() { + let mut u = super::UttSessions::default(); + u.started(11, 7, Some(1)); // in channel A + u.started(22, 1, None); // started outside a session + // Jarvis moves to B (session 2): A's utterance still belongs to session 1. + u.started(11, 8, Some(2)); + assert_eq!((u.session_of(11, 7), u.session_of(11, 8), u.session_of(22, 1)), (Some(1), Some(2), None)); + u.finished(11, 7); + assert_eq!(u.session_of(11, 7), None); + u.clear(); + assert_eq!(u.session_of(11, 8), None); + } + use super::*; #[tokio::test] diff --git a/app/src/discord_core.spec.md b/app/src/discord_core.spec.md index 77cf3d8..0a6012c 100644 --- a/app/src/discord_core.spec.md +++ b/app/src/discord_core.spec.md @@ -140,8 +140,11 @@ and the late "left" update cancelled the new join (seen on real Discord). `speaking_end` is sent. - When a person leaves the channel (gateway voice state or songbird's client disconnect), the bot sends `speaker_leave`, so their utterance finishes and the worker frees their place. -- **Events** for the layers above: `SpeechStart`, `Partial` and `Final` with the speaker's ID and - server display name; `SpeakingStopped` with the speaker's ID when their `speaking_end` is sent; +- **Events** for the layers above: `SpeechStart`, `Partial` and `Final` with the speaker's ID, + server display name and voice session. Each `Joined` carries a new session number (from 1); an + utterance belongs to the session Jarvis was in when its `speech_start` came (none outside a + session, and none once that session ended or the worker restarted), and its `Partial` and + `Final` carry that number even when they arrive later; `SpeakingStopped` with the speaker's ID when their `speaking_end` is sent; `SpeakStart` and `SpeakEnd`; `Stopped` when all speech is stopped (`stop()`, the slash command or the session ending), sent before the `SpeakEnd`s it causes; `Joined` and `Left` (with a reason: `command`, `moved`, `disconnected`, `shutdown`); `Addressed` and `Ranked` with the brain's answers to `ask_addressed` and `ask_rank`. `WorkerLost` when the voice worker connection ends: the finals of utterances in progress will not