fix(app): a transcript belongs to the voice session its utterance started in
Each Joined carries a new session number; the core records, when an utterance's speech_start arrives, the session Jarvis is in, and its Partial and Final carry that number (Heard.session) even when they arrive after a leave or a move. The dashboard shows only the current session's captions. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -108,8 +108,9 @@ reaches the screen.
|
||||
- `voice`: the voice channel Jarvis is in and the people there (ID and display name), as the core
|
||||
reports them to the worker.
|
||||
- `captions`: live captions, the last 30 lines: a speaker's partial transcript is replaced by
|
||||
their next partial and then by the final. Only while Jarvis is in a voice channel: a transcript
|
||||
that arrives after it left is not shown. On the screen only: never logged, never written,
|
||||
their next partial and then by the final. A transcript belongs to the voice session its
|
||||
utterance started in (`Heard.session`, `discord_core.spec.md`): only the current session's are
|
||||
shown, so one arriving after Jarvis left, or after it moved to another channel, is not. On the screen only: never logged, never written,
|
||||
cleared when Jarvis leaves the voice channel or the services stop.
|
||||
- `latency`: the last 20 voice answers, each split into its stages: speech end to final transcript,
|
||||
the brain (answer asked to answer text), the voice (answer text to the first sound) and the
|
||||
|
||||
@@ -314,8 +314,8 @@ mod tests {
|
||||
c.start().await.unwrap();
|
||||
let old = host.events.lock().unwrap().clone().unwrap();
|
||||
// Queued right before the stop, never handled before it.
|
||||
old.send(CoreEvent::Joined { guild_id: 1, channel_id: 77 }).unwrap();
|
||||
old.send(CoreEvent::Partial(crate::discord::Heard { speaker_id: 11, speaker_name: "민수".into(), utt: 1 },
|
||||
old.send(CoreEvent::Joined { guild_id: 1, channel_id: 77, session: 1 }).unwrap();
|
||||
old.send(CoreEvent::Partial(crate::discord::Heard { speaker_id: 11, speaker_name: "민수".into(), utt: 1, session: Some(1) },
|
||||
"자비스".into())).unwrap();
|
||||
c.stop().await.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(30)).await;
|
||||
@@ -324,7 +324,7 @@ mod tests {
|
||||
(json!("stopped"), Value::Null, json!([])));
|
||||
// After a new start, the earlier run's sender still reaches nothing.
|
||||
c.start().await.unwrap();
|
||||
let _ = old.send(CoreEvent::Joined { guild_id: 1, channel_id: 78 });
|
||||
let _ = old.send(CoreEvent::Joined { guild_id: 1, channel_id: 78, session: 2 });
|
||||
tokio::time::sleep(Duration::from_millis(30)).await;
|
||||
assert_eq!(c.snapshot().await["voiceChannel"], Value::Null);
|
||||
}
|
||||
@@ -335,8 +335,8 @@ mod tests {
|
||||
c.start().await.unwrap();
|
||||
let tx = host.events.lock().unwrap().clone().unwrap();
|
||||
tx.send(CoreEvent::Ready { user_id: 1, tag: "Jarvis#0001".into() }).unwrap();
|
||||
tx.send(CoreEvent::Joined { guild_id: 1, channel_id: 77 }).unwrap();
|
||||
tx.send(CoreEvent::Partial(crate::discord::Heard { speaker_id: 11, speaker_name: "민수".into(), utt: 1 },
|
||||
tx.send(CoreEvent::Joined { guild_id: 1, channel_id: 77, session: 1 }).unwrap();
|
||||
tx.send(CoreEvent::Partial(crate::discord::Heard { speaker_id: 11, speaker_name: "민수".into(), utt: 1, session: Some(1) },
|
||||
"자비스 안녕".into())).unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
let s = c.snapshot().await;
|
||||
|
||||
@@ -592,7 +592,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn partial(speaker: u64, text: &str) -> CoreEvent {
|
||||
CoreEvent::Partial(Heard { speaker_id: speaker, speaker_name: "x".into(), utt: 1 }, text.into())
|
||||
CoreEvent::Partial(Heard { speaker_id: speaker, speaker_name: "x".into(), utt: 1, session: Some(1) }, text.into())
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -622,7 +622,7 @@ mod tests {
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스 안녕".into())]);
|
||||
}
|
||||
fn fin(speaker: u64, text: &str, wait_ms: u64) -> CoreEvent {
|
||||
CoreEvent::Final(HeardFinal { heard: Heard { speaker_id: speaker, speaker_name: "x".into(), utt: 1 },
|
||||
CoreEvent::Final(HeardFinal { heard: Heard { speaker_id: speaker, speaker_name: "x".into(), utt: 1, session: Some(1) },
|
||||
text: text.into(), end_reason: "silence".into(), audio_ms: 1000, wait_ms, stt_ms: 100 })
|
||||
}
|
||||
fn end(id: &str, cancelled: bool) -> CoreEvent {
|
||||
@@ -633,11 +633,11 @@ mod tests {
|
||||
}
|
||||
|
||||
fn fin_utt(speaker: u64, utt: u32, text: &str) -> CoreEvent {
|
||||
CoreEvent::Final(HeardFinal { heard: Heard { speaker_id: speaker, speaker_name: "x".into(), utt },
|
||||
CoreEvent::Final(HeardFinal { heard: Heard { speaker_id: speaker, speaker_name: "x".into(), utt, session: Some(1) },
|
||||
text: text.into(), end_reason: "silence".into(), audio_ms: 1000, wait_ms: 100, stt_ms: 100 })
|
||||
}
|
||||
fn start(speaker: u64, utt: u32) -> CoreEvent {
|
||||
CoreEvent::SpeechStart(Heard { speaker_id: speaker, speaker_name: "x".into(), utt })
|
||||
CoreEvent::SpeechStart(Heard { speaker_id: speaker, speaker_name: "x".into(), utt, session: Some(1) })
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -53,6 +53,8 @@ struct Pending {
|
||||
pub struct Dashboard {
|
||||
bot_tag: Option<String>,
|
||||
voice_channel: Option<u64>,
|
||||
/// The voice session Jarvis is in (from `Joined`).
|
||||
session: Option<u64>,
|
||||
captions: VecDeque<Caption>,
|
||||
pending: HashMap<String, Pending>,
|
||||
answers: VecDeque<AnswerTiming>,
|
||||
@@ -66,16 +68,21 @@ impl Dashboard {
|
||||
pub fn on_event(&mut self, ev: &CoreEvent, now: Instant) {
|
||||
match ev {
|
||||
CoreEvent::Ready { tag, .. } => self.bot_tag = Some(tag.clone()),
|
||||
CoreEvent::Joined { channel_id, .. } => self.voice_channel = Some(*channel_id),
|
||||
CoreEvent::Joined { channel_id, session, .. } => {
|
||||
self.voice_channel = Some(*channel_id);
|
||||
self.session = Some(*session);
|
||||
}
|
||||
CoreEvent::Left { .. } => {
|
||||
self.voice_channel = None;
|
||||
self.session = None;
|
||||
self.captions.clear();
|
||||
self.pending.clear();
|
||||
}
|
||||
// Captions belong to the voice session: a transcript arriving after Jarvis left is dropped.
|
||||
CoreEvent::Partial(h, text) if self.voice_channel.is_some() =>
|
||||
// A transcript belongs to the session its utterance started in: one of an earlier session
|
||||
// (arriving after a leave, or after a move to another channel) is not shown.
|
||||
CoreEvent::Partial(h, text) if self.session.is_some() && h.session == self.session =>
|
||||
self.caption(h.speaker_id, &h.speaker_name, h.utt, text, false),
|
||||
CoreEvent::Final(f) if self.voice_channel.is_some() => {
|
||||
CoreEvent::Final(f) if self.session.is_some() && f.heard.session == self.session => {
|
||||
let h = &f.heard;
|
||||
self.caption(h.speaker_id, &h.speaker_name, h.utt, &f.text, true);
|
||||
}
|
||||
@@ -181,18 +188,24 @@ mod tests {
|
||||
use std::time::Duration;
|
||||
|
||||
const A: u64 = 11;
|
||||
fn heard_in(utt: u32, session: u64) -> Heard {
|
||||
Heard { speaker_id: A, speaker_name: "민수".into(), utt, session: Some(session) }
|
||||
}
|
||||
fn heard(utt: u32) -> Heard {
|
||||
Heard { speaker_id: A, speaker_name: "민수".into(), utt }
|
||||
heard_in(utt, 1)
|
||||
}
|
||||
fn fin_in(utt: u32, text: &str, wait_ms: u64, session: u64) -> CoreEvent {
|
||||
CoreEvent::Final(HeardFinal { heard: heard_in(utt, session), text: text.into(), end_reason: "silence".into(),
|
||||
audio_ms: 900, wait_ms, stt_ms: 200 })
|
||||
}
|
||||
fn fin(utt: u32, text: &str, wait_ms: u64) -> CoreEvent {
|
||||
CoreEvent::Final(HeardFinal { heard: heard(utt), text: text.into(), end_reason: "silence".into(),
|
||||
audio_ms: 900, wait_ms, stt_ms: 200 })
|
||||
fin_in(utt, text, wait_ms, 1)
|
||||
}
|
||||
fn at(t: Instant, ms: u64) -> Instant {
|
||||
t + Duration::from_millis(ms)
|
||||
}
|
||||
fn joined() -> CoreEvent {
|
||||
CoreEvent::Joined { guild_id: 1, channel_id: 5 }
|
||||
CoreEvent::Joined { guild_id: 1, channel_id: 5, session: 1 }
|
||||
}
|
||||
fn left() -> CoreEvent {
|
||||
CoreEvent::Left { guild_id: 1, channel_id: 5, reason: "command".into() }
|
||||
@@ -241,6 +254,25 @@ mod tests {
|
||||
assert_eq!(d.view(&[], t)["voiceChannel"], Value::Null);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_transcript_of_the_channel_jarvis_moved_away_from_stays_out_of_the_new_ones_captions() {
|
||||
let (mut d, t) = (Dashboard::default(), Instant::now());
|
||||
d.on_event(&joined(), t);
|
||||
d.on_event(&CoreEvent::SpeechStart(heard_in(7, 1)), t); // started in channel A, session 1
|
||||
d.on_event(&left(), t);
|
||||
d.on_event(&CoreEvent::Joined { guild_id: 1, channel_id: 6, session: 2 }, t);
|
||||
d.on_event(&CoreEvent::Partial(heard_in(7, 1), "A에서 한 말".into()), t);
|
||||
d.on_event(&fin_in(7, "A에서 한 말", 300, 1), t);
|
||||
assert!(d.captions().is_empty(), "session 1's utterance is not session 2's");
|
||||
// An utterance whose start was not seen in a session (None) is not shown either.
|
||||
d.on_event(&CoreEvent::Partial(Heard { speaker_id: A, speaker_name: "민수".into(), utt: 8, session: None },
|
||||
"어디서 온 말".into()), t);
|
||||
assert!(d.captions().is_empty());
|
||||
// The same person speaking in B is.
|
||||
d.on_event(&fin_in(9, "B에서 한 말", 300, 2), t);
|
||||
assert_eq!(d.captions().iter().map(|c| c.text.as_str()).collect::<Vec<_>>(), vec!["B에서 한 말"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_answer_is_timed_from_its_own_requests_speech_end_to_its_first_sound() {
|
||||
let (mut d, t) = (Dashboard::default(), Instant::now());
|
||||
|
||||
@@ -39,6 +39,10 @@ pub struct Heard {
|
||||
pub speaker_id: u64,
|
||||
pub speaker_name: String,
|
||||
pub utt: u32,
|
||||
/// The voice session the utterance started in (the number its `Joined` carried); None when its
|
||||
/// start was not seen in a session. A transcript belongs to the session its utterance started
|
||||
/// in, even when it arrives after Jarvis left or moved on.
|
||||
pub session: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -54,7 +58,8 @@ pub struct HeardFinal {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum CoreEvent {
|
||||
Ready { user_id: u64, tag: String },
|
||||
Joined { guild_id: u64, channel_id: u64 },
|
||||
/// Jarvis joined a voice channel; `session` numbers each voice session from 1.
|
||||
Joined { guild_id: u64, channel_id: u64, session: u64 },
|
||||
Left { guild_id: u64, channel_id: u64, reason: String },
|
||||
SpeechStart(Heard),
|
||||
/// The speaker stopped sending audio (their final is still to come).
|
||||
@@ -191,6 +196,34 @@ struct Inner {
|
||||
/// Serialises join and leave.
|
||||
voice_lock: tokio::sync::Mutex<()>,
|
||||
names: Mutex<HashMap<u64, String>>,
|
||||
/// The latest voice session's number, and the session each utterance in progress started in.
|
||||
voice_session: std::sync::atomic::AtomicU64,
|
||||
utt_sessions: Mutex<UttSessions>,
|
||||
}
|
||||
|
||||
/// The voice session each utterance in progress started in (`Heard::session`).
|
||||
#[derive(Default)]
|
||||
pub struct UttSessions(HashMap<(u64, u32), u64>);
|
||||
|
||||
impl UttSessions {
|
||||
/// Utterance `(speaker, utt)` started; `current` is the session Jarvis is in (None: none).
|
||||
pub fn started(&mut self, speaker: u64, utt: u32, current: Option<u64>) {
|
||||
match current {
|
||||
Some(s) => self.0.insert((speaker, utt), s),
|
||||
None => self.0.remove(&(speaker, utt)),
|
||||
};
|
||||
}
|
||||
pub fn session_of(&self, speaker: u64, utt: u32) -> Option<u64> {
|
||||
self.0.get(&(speaker, utt)).copied()
|
||||
}
|
||||
/// Its final came: nothing more of it will.
|
||||
pub fn finished(&mut self, speaker: u64, utt: u32) {
|
||||
self.0.remove(&(speaker, utt));
|
||||
}
|
||||
/// The session ended or the worker restarted: no utterance in progress belongs to a session now.
|
||||
pub fn clear(&mut self) {
|
||||
self.0.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle to the core; cheap to clone.
|
||||
@@ -225,6 +258,8 @@ impl DiscordCore {
|
||||
session: Mutex::new(None),
|
||||
voice_lock: tokio::sync::Mutex::new(()),
|
||||
names: Mutex::new(HashMap::new()),
|
||||
voice_session: std::sync::atomic::AtomicU64::new(0),
|
||||
utt_sessions: Mutex::new(UttSessions::default()),
|
||||
});
|
||||
tokio::spawn(player_task(Arc::downgrade(&inner), player_rx));
|
||||
tokio::spawn(worker_task(Arc::downgrade(&inner), worker_events));
|
||||
@@ -391,11 +426,13 @@ impl DiscordCore {
|
||||
}
|
||||
let alive = Arc::new(AtomicBool::new(true));
|
||||
tokio::spawn(speaking_end_timer(Arc::downgrade(&self.inner), rx.clone(), alive.clone()));
|
||||
// A new session number before any of its utterances can start.
|
||||
let session = self.inner.voice_session.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
*self.inner.session.lock().unwrap() = Some(Session { guild_id: gc.guild_id.get(), channel_id, rx, alive });
|
||||
let _ = self.inner.player.send(PlayerCmd::SetCall(Some(call)));
|
||||
report_voice_state(&self.inner);
|
||||
debug_log!("voice", "joined voice {channel_id}");
|
||||
let _ = self.inner.events.send(CoreEvent::Joined { guild_id: gc.guild_id.get(), channel_id });
|
||||
let _ = self.inner.events.send(CoreEvent::Joined { guild_id: gc.guild_id.get(), channel_id, session });
|
||||
let notice = CreateMessage::new().content(&self.inner.settings.voice.join_notice)
|
||||
.allowed_mentions(CreateAllowedMentions::new());
|
||||
if let Err(e) = gc.id.send_message(&http, notice).await {
|
||||
@@ -615,6 +652,8 @@ fn with_playback<T>(inner: &Inner, f: impl FnOnce(&mut Playback<WorkerClient, Pl
|
||||
/// the connection and points it at the new channel instead).
|
||||
async fn end_session(inner: &Arc<Inner>, s: Session, reason: &str, disconnect: bool) {
|
||||
s.alive.store(false, Ordering::Relaxed);
|
||||
// Utterances still in progress carry no session from here: their late transcripts belong to none.
|
||||
inner.utt_sessions.lock().unwrap().clear();
|
||||
let _ = inner.events.send(CoreEvent::Stopped);
|
||||
with_playback(inner, |p| p.stop());
|
||||
let _ = inner.player.send(PlayerCmd::SetCall(None));
|
||||
@@ -693,7 +732,15 @@ fn name_of(inner: &Inner, user: u64) -> String {
|
||||
|
||||
fn heard(inner: &Inner, speaker: &str, utt: u32) -> Option<Heard> {
|
||||
let id: u64 = speaker.parse().ok()?;
|
||||
Some(Heard { speaker_id: id, speaker_name: name_of(inner, id), utt })
|
||||
let session = inner.utt_sessions.lock().unwrap().session_of(id, utt);
|
||||
Some(Heard { speaker_id: id, speaker_name: name_of(inner, id), utt, session })
|
||||
}
|
||||
|
||||
/// An utterance started: it belongs to the session Jarvis is in now (none outside one).
|
||||
fn utterance_started(inner: &Inner, speaker: &str, utt: u32) {
|
||||
let Ok(id) = speaker.parse::<u64>() else { return };
|
||||
let current = inner.session.lock().unwrap().is_some().then(|| inner.voice_session.load(Ordering::SeqCst));
|
||||
inner.utt_sessions.lock().unwrap().started(id, utt, current);
|
||||
}
|
||||
|
||||
fn to_chat(inner: &Inner, m: &Message) -> ChatMessage {
|
||||
@@ -716,6 +763,7 @@ async fn worker_task(core: std::sync::Weak<Inner>, mut rx: mpsc::UnboundedReceiv
|
||||
let Some(inner) = core.upgrade() else { return };
|
||||
match ev {
|
||||
WorkerEvent::SpeechStart(m) => {
|
||||
utterance_started(&inner, &m.speaker, m.utt);
|
||||
if let Some(h) = heard(&inner, &m.speaker, m.utt) {
|
||||
let _ = inner.events.send(CoreEvent::SpeechStart(h));
|
||||
}
|
||||
@@ -753,6 +801,8 @@ async fn worker_task(core: std::sync::Weak<Inner>, mut rx: mpsc::UnboundedReceiv
|
||||
});
|
||||
}
|
||||
WorkerEvent::Disconnected => {
|
||||
// The restarted worker numbers utterances afresh: the old ones' sessions are void.
|
||||
inner.utt_sessions.lock().unwrap().clear();
|
||||
let _ = inner.events.send(CoreEvent::WorkerLost);
|
||||
}
|
||||
WorkerEvent::Error(_) => {}
|
||||
@@ -763,7 +813,11 @@ async fn worker_task(core: std::sync::Weak<Inner>, mut rx: mpsc::UnboundedReceiv
|
||||
fn on_final(inner: &Inner, m: Final) {
|
||||
debug_log!("voice", "final speaker={} utt={} reason={} chars={} audio_ms={} wait_ms={}", m.speaker, m.utt,
|
||||
m.end_reason, m.text.chars().count(), m.audio_ms, m.wait_ms);
|
||||
if let Some(h) = heard(inner, &m.speaker, m.utt) {
|
||||
let h = heard(inner, &m.speaker, m.utt);
|
||||
if let Ok(id) = m.speaker.parse::<u64>() {
|
||||
inner.utt_sessions.lock().unwrap().finished(id, m.utt);
|
||||
}
|
||||
if let Some(h) = h {
|
||||
let _ = inner.events.send(CoreEvent::Final(HeardFinal { heard: h, text: m.text, end_reason: m.end_reason,
|
||||
audio_ms: m.audio_ms, wait_ms: m.wait_ms, stt_ms: m.stt_ms }));
|
||||
}
|
||||
@@ -1090,6 +1144,20 @@ impl Handler {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn an_utterance_keeps_the_session_it_started_in() {
|
||||
let mut u = super::UttSessions::default();
|
||||
u.started(11, 7, Some(1)); // in channel A
|
||||
u.started(22, 1, None); // started outside a session
|
||||
// Jarvis moves to B (session 2): A's utterance still belongs to session 1.
|
||||
u.started(11, 8, Some(2));
|
||||
assert_eq!((u.session_of(11, 7), u.session_of(11, 8), u.session_of(22, 1)), (Some(1), Some(2), None));
|
||||
u.finished(11, 7);
|
||||
assert_eq!(u.session_of(11, 7), None);
|
||||
u.clear();
|
||||
assert_eq!(u.session_of(11, 8), None);
|
||||
}
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -140,8 +140,11 @@ and the late "left" update cancelled the new join (seen on real Discord).
|
||||
`speaking_end` is sent.
|
||||
- When a person leaves the channel (gateway voice state or songbird's client disconnect), the bot
|
||||
sends `speaker_leave`, so their utterance finishes and the worker frees their place.
|
||||
- **Events** for the layers above: `SpeechStart`, `Partial` and `Final` with the speaker's ID and
|
||||
server display name; `SpeakingStopped` with the speaker's ID when their `speaking_end` is sent;
|
||||
- **Events** for the layers above: `SpeechStart`, `Partial` and `Final` with the speaker's ID,
|
||||
server display name and voice session. Each `Joined` carries a new session number (from 1); an
|
||||
utterance belongs to the session Jarvis was in when its `speech_start` came (none outside a
|
||||
session, and none once that session ended or the worker restarted), and its `Partial` and
|
||||
`Final` carry that number even when they arrive later; `SpeakingStopped` with the speaker's ID when their `speaking_end` is sent;
|
||||
`SpeakStart` and `SpeakEnd`; `Stopped` when all speech is stopped (`stop()`, the slash command or the
|
||||
session ending), sent before the `SpeakEnd`s it causes; `Joined` and `Left` (with a reason: `command`,
|
||||
`moved`, `disconnected`, `shutdown`); `Addressed` and `Ranked` with the brain's answers to `ask_addressed` and `ask_rank`. `WorkerLost` when the voice worker connection ends: the finals of utterances in progress will not
|
||||
|
||||
Reference in New Issue
Block a user