fix(app): a transcript belongs to the voice session its utterance started in

Each Joined carries a new session number; the core records, when an
utterance's speech_start arrives, the session Jarvis is in, and its Partial
and Final carry that number (Heard.session) even when they arrive after a
leave or a move. The dashboard shows only the current session's captions.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-12 14:00:21 +09:00
parent 371e87f553
commit e8ddde5be6
6 changed files with 129 additions and 25 deletions

View File

@@ -108,8 +108,9 @@ reaches the screen.
- `voice`: the voice channel Jarvis is in and the people there (ID and display name), as the core
reports them to the worker.
- `captions`: live captions, the last 30 lines: a speaker's partial transcript is replaced by
their next partial and then by the final. Only while Jarvis is in a voice channel: a transcript
that arrives after it left is not shown. On the screen only: never logged, never written,
their next partial and then by the final. A transcript belongs to the voice session its
utterance started in (`Heard.session`, `discord_core.spec.md`): only the current session's are
shown, so one arriving after Jarvis left, or after it moved to another channel, is not. On the screen only: never logged, never written,
cleared when Jarvis leaves the voice channel or the services stop.
- `latency`: the last 20 voice answers, each split into its stages: speech end to final transcript,
the brain (answer asked to answer text), the voice (answer text to the first sound) and the

View File

@@ -314,8 +314,8 @@ mod tests {
c.start().await.unwrap();
let old = host.events.lock().unwrap().clone().unwrap();
// Queued right before the stop, never handled before it.
old.send(CoreEvent::Joined { guild_id: 1, channel_id: 77 }).unwrap();
old.send(CoreEvent::Partial(crate::discord::Heard { speaker_id: 11, speaker_name: "민수".into(), utt: 1 },
old.send(CoreEvent::Joined { guild_id: 1, channel_id: 77, session: 1 }).unwrap();
old.send(CoreEvent::Partial(crate::discord::Heard { speaker_id: 11, speaker_name: "민수".into(), utt: 1, session: Some(1) },
"자비스".into())).unwrap();
c.stop().await.unwrap();
tokio::time::sleep(Duration::from_millis(30)).await;
@@ -324,7 +324,7 @@ mod tests {
(json!("stopped"), Value::Null, json!([])));
// After a new start, the earlier run's sender still reaches nothing.
c.start().await.unwrap();
let _ = old.send(CoreEvent::Joined { guild_id: 1, channel_id: 78 });
let _ = old.send(CoreEvent::Joined { guild_id: 1, channel_id: 78, session: 2 });
tokio::time::sleep(Duration::from_millis(30)).await;
assert_eq!(c.snapshot().await["voiceChannel"], Value::Null);
}
@@ -335,8 +335,8 @@ mod tests {
c.start().await.unwrap();
let tx = host.events.lock().unwrap().clone().unwrap();
tx.send(CoreEvent::Ready { user_id: 1, tag: "Jarvis#0001".into() }).unwrap();
tx.send(CoreEvent::Joined { guild_id: 1, channel_id: 77 }).unwrap();
tx.send(CoreEvent::Partial(crate::discord::Heard { speaker_id: 11, speaker_name: "민수".into(), utt: 1 },
tx.send(CoreEvent::Joined { guild_id: 1, channel_id: 77, session: 1 }).unwrap();
tx.send(CoreEvent::Partial(crate::discord::Heard { speaker_id: 11, speaker_name: "민수".into(), utt: 1, session: Some(1) },
"자비스 안녕".into())).unwrap();
tokio::time::sleep(Duration::from_millis(50)).await;
let s = c.snapshot().await;

View File

@@ -592,7 +592,7 @@ mod tests {
}
fn partial(speaker: u64, text: &str) -> CoreEvent {
CoreEvent::Partial(Heard { speaker_id: speaker, speaker_name: "x".into(), utt: 1 }, text.into())
CoreEvent::Partial(Heard { speaker_id: speaker, speaker_name: "x".into(), utt: 1, session: Some(1) }, text.into())
}
#[test]
@@ -622,7 +622,7 @@ mod tests {
assert_eq!(f.take(), vec![Did::Answer(A, "자비스 안녕".into())]);
}
fn fin(speaker: u64, text: &str, wait_ms: u64) -> CoreEvent {
CoreEvent::Final(HeardFinal { heard: Heard { speaker_id: speaker, speaker_name: "x".into(), utt: 1 },
CoreEvent::Final(HeardFinal { heard: Heard { speaker_id: speaker, speaker_name: "x".into(), utt: 1, session: Some(1) },
text: text.into(), end_reason: "silence".into(), audio_ms: 1000, wait_ms, stt_ms: 100 })
}
fn end(id: &str, cancelled: bool) -> CoreEvent {
@@ -633,11 +633,11 @@ mod tests {
}
fn fin_utt(speaker: u64, utt: u32, text: &str) -> CoreEvent {
CoreEvent::Final(HeardFinal { heard: Heard { speaker_id: speaker, speaker_name: "x".into(), utt },
CoreEvent::Final(HeardFinal { heard: Heard { speaker_id: speaker, speaker_name: "x".into(), utt, session: Some(1) },
text: text.into(), end_reason: "silence".into(), audio_ms: 1000, wait_ms: 100, stt_ms: 100 })
}
fn start(speaker: u64, utt: u32) -> CoreEvent {
CoreEvent::SpeechStart(Heard { speaker_id: speaker, speaker_name: "x".into(), utt })
CoreEvent::SpeechStart(Heard { speaker_id: speaker, speaker_name: "x".into(), utt, session: Some(1) })
}
#[test]

View File

@@ -53,6 +53,8 @@ struct Pending {
pub struct Dashboard {
bot_tag: Option<String>,
voice_channel: Option<u64>,
/// The voice session Jarvis is in (from `Joined`).
session: Option<u64>,
captions: VecDeque<Caption>,
pending: HashMap<String, Pending>,
answers: VecDeque<AnswerTiming>,
@@ -66,16 +68,21 @@ impl Dashboard {
pub fn on_event(&mut self, ev: &CoreEvent, now: Instant) {
match ev {
CoreEvent::Ready { tag, .. } => self.bot_tag = Some(tag.clone()),
CoreEvent::Joined { channel_id, .. } => self.voice_channel = Some(*channel_id),
CoreEvent::Joined { channel_id, session, .. } => {
self.voice_channel = Some(*channel_id);
self.session = Some(*session);
}
CoreEvent::Left { .. } => {
self.voice_channel = None;
self.session = None;
self.captions.clear();
self.pending.clear();
}
// Captions belong to the voice session: a transcript arriving after Jarvis left is dropped.
CoreEvent::Partial(h, text) if self.voice_channel.is_some() =>
// A transcript belongs to the session its utterance started in: one of an earlier session
// (arriving after a leave, or after a move to another channel) is not shown.
CoreEvent::Partial(h, text) if self.session.is_some() && h.session == self.session =>
self.caption(h.speaker_id, &h.speaker_name, h.utt, text, false),
CoreEvent::Final(f) if self.voice_channel.is_some() => {
CoreEvent::Final(f) if self.session.is_some() && f.heard.session == self.session => {
let h = &f.heard;
self.caption(h.speaker_id, &h.speaker_name, h.utt, &f.text, true);
}
@@ -181,18 +188,24 @@ mod tests {
use std::time::Duration;
const A: u64 = 11;
fn heard_in(utt: u32, session: u64) -> Heard {
Heard { speaker_id: A, speaker_name: "민수".into(), utt, session: Some(session) }
}
fn heard(utt: u32) -> Heard {
Heard { speaker_id: A, speaker_name: "민수".into(), utt }
heard_in(utt, 1)
}
fn fin_in(utt: u32, text: &str, wait_ms: u64, session: u64) -> CoreEvent {
CoreEvent::Final(HeardFinal { heard: heard_in(utt, session), text: text.into(), end_reason: "silence".into(),
audio_ms: 900, wait_ms, stt_ms: 200 })
}
fn fin(utt: u32, text: &str, wait_ms: u64) -> CoreEvent {
CoreEvent::Final(HeardFinal { heard: heard(utt), text: text.into(), end_reason: "silence".into(),
audio_ms: 900, wait_ms, stt_ms: 200 })
fin_in(utt, text, wait_ms, 1)
}
fn at(t: Instant, ms: u64) -> Instant {
t + Duration::from_millis(ms)
}
fn joined() -> CoreEvent {
CoreEvent::Joined { guild_id: 1, channel_id: 5 }
CoreEvent::Joined { guild_id: 1, channel_id: 5, session: 1 }
}
fn left() -> CoreEvent {
CoreEvent::Left { guild_id: 1, channel_id: 5, reason: "command".into() }
@@ -241,6 +254,25 @@ mod tests {
assert_eq!(d.view(&[], t)["voiceChannel"], Value::Null);
}
#[test]
fn a_transcript_of_the_channel_jarvis_moved_away_from_stays_out_of_the_new_ones_captions() {
let (mut d, t) = (Dashboard::default(), Instant::now());
d.on_event(&joined(), t);
d.on_event(&CoreEvent::SpeechStart(heard_in(7, 1)), t); // started in channel A, session 1
d.on_event(&left(), t);
d.on_event(&CoreEvent::Joined { guild_id: 1, channel_id: 6, session: 2 }, t);
d.on_event(&CoreEvent::Partial(heard_in(7, 1), "A에서 한 말".into()), t);
d.on_event(&fin_in(7, "A에서 한 말", 300, 1), t);
assert!(d.captions().is_empty(), "session 1's utterance is not session 2's");
// An utterance whose start was not seen in a session (None) is not shown either.
d.on_event(&CoreEvent::Partial(Heard { speaker_id: A, speaker_name: "민수".into(), utt: 8, session: None },
"어디서 온 말".into()), t);
assert!(d.captions().is_empty());
// The same person speaking in B is.
d.on_event(&fin_in(9, "B에서 한 말", 300, 2), t);
assert_eq!(d.captions().iter().map(|c| c.text.as_str()).collect::<Vec<_>>(), vec!["B에서 한 말"]);
}
#[test]
fn an_answer_is_timed_from_its_own_requests_speech_end_to_its_first_sound() {
let (mut d, t) = (Dashboard::default(), Instant::now());

View File

@@ -39,6 +39,10 @@ pub struct Heard {
pub speaker_id: u64,
pub speaker_name: String,
pub utt: u32,
/// The voice session the utterance started in (the number its `Joined` carried); None when its
/// start was not seen in a session. A transcript belongs to the session its utterance started
/// in, even when it arrives after Jarvis left or moved on.
pub session: Option<u64>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -54,7 +58,8 @@ pub struct HeardFinal {
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CoreEvent {
Ready { user_id: u64, tag: String },
Joined { guild_id: u64, channel_id: u64 },
/// Jarvis joined a voice channel; `session` numbers each voice session from 1.
Joined { guild_id: u64, channel_id: u64, session: u64 },
Left { guild_id: u64, channel_id: u64, reason: String },
SpeechStart(Heard),
/// The speaker stopped sending audio (their final is still to come).
@@ -191,6 +196,34 @@ struct Inner {
/// Serialises join and leave.
voice_lock: tokio::sync::Mutex<()>,
names: Mutex<HashMap<u64, String>>,
/// The latest voice session's number, and the session each utterance in progress started in.
voice_session: std::sync::atomic::AtomicU64,
utt_sessions: Mutex<UttSessions>,
}
/// The voice session each utterance in progress started in (`Heard::session`).
#[derive(Default)]
pub struct UttSessions(HashMap<(u64, u32), u64>);
impl UttSessions {
/// Utterance `(speaker, utt)` started; `current` is the session Jarvis is in (None: none).
pub fn started(&mut self, speaker: u64, utt: u32, current: Option<u64>) {
match current {
Some(s) => self.0.insert((speaker, utt), s),
None => self.0.remove(&(speaker, utt)),
};
}
pub fn session_of(&self, speaker: u64, utt: u32) -> Option<u64> {
self.0.get(&(speaker, utt)).copied()
}
/// Its final came: nothing more of it will.
pub fn finished(&mut self, speaker: u64, utt: u32) {
self.0.remove(&(speaker, utt));
}
/// The session ended or the worker restarted: no utterance in progress belongs to a session now.
pub fn clear(&mut self) {
self.0.clear();
}
}
/// Handle to the core; cheap to clone.
@@ -225,6 +258,8 @@ impl DiscordCore {
session: Mutex::new(None),
voice_lock: tokio::sync::Mutex::new(()),
names: Mutex::new(HashMap::new()),
voice_session: std::sync::atomic::AtomicU64::new(0),
utt_sessions: Mutex::new(UttSessions::default()),
});
tokio::spawn(player_task(Arc::downgrade(&inner), player_rx));
tokio::spawn(worker_task(Arc::downgrade(&inner), worker_events));
@@ -391,11 +426,13 @@ impl DiscordCore {
}
let alive = Arc::new(AtomicBool::new(true));
tokio::spawn(speaking_end_timer(Arc::downgrade(&self.inner), rx.clone(), alive.clone()));
// A new session number before any of its utterances can start.
let session = self.inner.voice_session.fetch_add(1, Ordering::SeqCst) + 1;
*self.inner.session.lock().unwrap() = Some(Session { guild_id: gc.guild_id.get(), channel_id, rx, alive });
let _ = self.inner.player.send(PlayerCmd::SetCall(Some(call)));
report_voice_state(&self.inner);
debug_log!("voice", "joined voice {channel_id}");
let _ = self.inner.events.send(CoreEvent::Joined { guild_id: gc.guild_id.get(), channel_id });
let _ = self.inner.events.send(CoreEvent::Joined { guild_id: gc.guild_id.get(), channel_id, session });
let notice = CreateMessage::new().content(&self.inner.settings.voice.join_notice)
.allowed_mentions(CreateAllowedMentions::new());
if let Err(e) = gc.id.send_message(&http, notice).await {
@@ -615,6 +652,8 @@ fn with_playback<T>(inner: &Inner, f: impl FnOnce(&mut Playback<WorkerClient, Pl
/// the connection and points it at the new channel instead).
async fn end_session(inner: &Arc<Inner>, s: Session, reason: &str, disconnect: bool) {
s.alive.store(false, Ordering::Relaxed);
// Utterances still in progress carry no session from here: their late transcripts belong to none.
inner.utt_sessions.lock().unwrap().clear();
let _ = inner.events.send(CoreEvent::Stopped);
with_playback(inner, |p| p.stop());
let _ = inner.player.send(PlayerCmd::SetCall(None));
@@ -693,7 +732,15 @@ fn name_of(inner: &Inner, user: u64) -> String {
fn heard(inner: &Inner, speaker: &str, utt: u32) -> Option<Heard> {
let id: u64 = speaker.parse().ok()?;
Some(Heard { speaker_id: id, speaker_name: name_of(inner, id), utt })
let session = inner.utt_sessions.lock().unwrap().session_of(id, utt);
Some(Heard { speaker_id: id, speaker_name: name_of(inner, id), utt, session })
}
/// An utterance started: it belongs to the session Jarvis is in now (none outside one).
fn utterance_started(inner: &Inner, speaker: &str, utt: u32) {
let Ok(id) = speaker.parse::<u64>() else { return };
let current = inner.session.lock().unwrap().is_some().then(|| inner.voice_session.load(Ordering::SeqCst));
inner.utt_sessions.lock().unwrap().started(id, utt, current);
}
fn to_chat(inner: &Inner, m: &Message) -> ChatMessage {
@@ -716,6 +763,7 @@ async fn worker_task(core: std::sync::Weak<Inner>, mut rx: mpsc::UnboundedReceiv
let Some(inner) = core.upgrade() else { return };
match ev {
WorkerEvent::SpeechStart(m) => {
utterance_started(&inner, &m.speaker, m.utt);
if let Some(h) = heard(&inner, &m.speaker, m.utt) {
let _ = inner.events.send(CoreEvent::SpeechStart(h));
}
@@ -753,6 +801,8 @@ async fn worker_task(core: std::sync::Weak<Inner>, mut rx: mpsc::UnboundedReceiv
});
}
WorkerEvent::Disconnected => {
// The restarted worker numbers utterances afresh: the old ones' sessions are void.
inner.utt_sessions.lock().unwrap().clear();
let _ = inner.events.send(CoreEvent::WorkerLost);
}
WorkerEvent::Error(_) => {}
@@ -763,7 +813,11 @@ async fn worker_task(core: std::sync::Weak<Inner>, mut rx: mpsc::UnboundedReceiv
fn on_final(inner: &Inner, m: Final) {
debug_log!("voice", "final speaker={} utt={} reason={} chars={} audio_ms={} wait_ms={}", m.speaker, m.utt,
m.end_reason, m.text.chars().count(), m.audio_ms, m.wait_ms);
if let Some(h) = heard(inner, &m.speaker, m.utt) {
let h = heard(inner, &m.speaker, m.utt);
if let Ok(id) = m.speaker.parse::<u64>() {
inner.utt_sessions.lock().unwrap().finished(id, m.utt);
}
if let Some(h) = h {
let _ = inner.events.send(CoreEvent::Final(HeardFinal { heard: h, text: m.text, end_reason: m.end_reason,
audio_ms: m.audio_ms, wait_ms: m.wait_ms, stt_ms: m.stt_ms }));
}
@@ -1090,6 +1144,20 @@ impl Handler {
#[cfg(test)]
mod tests {
#[test]
fn an_utterance_keeps_the_session_it_started_in() {
let mut u = super::UttSessions::default();
u.started(11, 7, Some(1)); // in channel A
u.started(22, 1, None); // started outside a session
// Jarvis moves to B (session 2): A's utterance still belongs to session 1.
u.started(11, 8, Some(2));
assert_eq!((u.session_of(11, 7), u.session_of(11, 8), u.session_of(22, 1)), (Some(1), Some(2), None));
u.finished(11, 7);
assert_eq!(u.session_of(11, 7), None);
u.clear();
assert_eq!(u.session_of(11, 8), None);
}
use super::*;
#[tokio::test]

View File

@@ -140,8 +140,11 @@ and the late "left" update cancelled the new join (seen on real Discord).
`speaking_end` is sent.
- When a person leaves the channel (gateway voice state or songbird's client disconnect), the bot
sends `speaker_leave`, so their utterance finishes and the worker frees their place.
- **Events** for the layers above: `SpeechStart`, `Partial` and `Final` with the speaker's ID and
server display name; `SpeakingStopped` with the speaker's ID when their `speaking_end` is sent;
- **Events** for the layers above: `SpeechStart`, `Partial` and `Final` with the speaker's ID,
server display name and voice session. Each `Joined` carries a new session number (from 1); an
utterance belongs to the session Jarvis was in when its `speech_start` came (none outside a
session, and none once that session ended or the worker restarted), and its `Partial` and
`Final` carry that number even when they arrive later; `SpeakingStopped` with the speaker's ID when their `speaking_end` is sent;
`SpeakStart` and `SpeakEnd`; `Stopped` when all speech is stopped (`stop()`, the slash command or the
session ending), sent before the `SpeakEnd`s it causes; `Joined` and `Left` (with a reason: `command`,
`moved`, `disconnected`, `shutdown`); `Addressed` and `Ranked` with the brain's answers to `ask_addressed` and `ask_rank`. `WorkerLost` when the voice worker connection ends: the finals of utterances in progress will not