//! The conversation loop (`app/src/conversation.spec.md`): finished utterances go through call //! detection and the response policy, and the policy's actions become brain answers, stops and //! missed-request records. //! //! `Conversation` holds no Discord or worker types: it talks to the voice side through `Voice`, so the //! glue is tested with a fake. `run` drives it from the core's events with a timer for the policy. use std::collections::{HashMap, HashSet, VecDeque}; use std::time::{Duration, Instant}; use tokio::sync::mpsc; use crate::calls::{Call, CallSettings}; use crate::discord::{CoreEvent, DiscordCore}; use crate::policy::{Action, MissReason, Policy, PolicySettings, Request}; use crate::debug_log; /// How long the brain's "was it said to Jarvis?" answer is waited for; no answer counts as no. pub const ADDRESSED_TIMEOUT: Duration = Duration::from_secs(5); /// Missed requests kept for the dashboard (M5). pub const MISSED_KEPT: usize = 100; /// What the conversation needs from the voice side. pub trait Voice { /// Have the brain answer and speak it; the speech request id, or None when it cannot. `heard`: /// when the request's speech ended and its final arrived (the dashboard times the answer by it). fn answer(&self, user: u64, text: &str, heard: (Instant, Instant)) -> Option; /// Stop one speech request. fn cancel(&self, id: &str); /// Stop all speech at once. fn stop_all(&self); /// Play the earcon now (queued like speech; dropped by a stop). fn earcon(&self); /// A call is still being said (partial transcript): the brain may get ready to answer it. fn prepare(&self, user: u64, text: &str); fn ask_addressed(&self, id: &str, user: u64, text: &str) -> bool; fn ask_rank(&self, id: &str, candidates: &[(String, u64, String)]) -> bool; /// Have the brain answer a chat message (not spoken); its text comes as `CoreEvent::ReplyText`. fn chat_answer(&self, id: &str, user: u64, channel: u64, text: &str) -> bool; /// Post the chat answer `id` in `channel` as a reply to `message` (never pinging anyone), then /// tell the brain whether it was posted. fn send_reply(&self, id: &str, channel: u64, message: u64, text: &str); /// Tell the brain the chat answer `id` was not posted (late, failed or empty). fn reply_not_posted(&self, id: &str); } impl Voice for DiscordCore { fn answer(&self, user: u64, text: &str, heard: (Instant, Instant)) -> Option { DiscordCore::answer(self, user, text, heard) } fn cancel(&self, id: &str) { DiscordCore::cancel(self, id) } fn stop_all(&self) { DiscordCore::stop(self); } fn earcon(&self) { DiscordCore::earcon(self); } fn prepare(&self, user: u64, text: &str) { DiscordCore::prepare(self, user, text); } fn ask_addressed(&self, id: &str, user: u64, text: &str) -> bool { DiscordCore::ask_addressed(self, id, user, text) } fn ask_rank(&self, id: &str, candidates: &[(String, u64, String)]) -> bool { DiscordCore::ask_rank(self, id, candidates) } fn chat_answer(&self, id: &str, user: u64, channel: u64, text: &str) -> bool { DiscordCore::chat_answer(self, id, user, channel, text) } fn send_reply(&self, id: &str, channel: u64, message: u64, text: &str) { let core = self.clone(); let (id, text) = (id.to_owned(), text.to_owned()); tokio::spawn(async move { let posted = core.send_chat_reply(channel, message, &text).await; if let Err(e) = &posted { crate::error_log!("conversation", "could not post the chat answer in {channel}: {e}"); } core.reply_delivered(&id, posted.is_ok()); }); } fn reply_not_posted(&self, id: &str) { self.reply_delivered(id, false); } } /// How a registered chat channel is answered (`chatChannels[].answer`). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ChatChannelCall { /// Answer every message, not only those with a wake word. pub everything: bool, /// Sending is on; without it no answer could be posted, so none is asked for. pub send: bool, } /// Call detection for chat: the chat wake words (`call.chatWakeWords`, else the voice ones) with the /// shared designated-people switches, and each registered channel's answer setting. #[derive(Debug, Clone, Default)] pub struct ChatCalls { pub calls: CallSettings, pub channels: HashMap, } impl ChatCalls { pub fn from_settings(s: &crate::settings::Settings) -> Self { let mut calls = s.call.clone(); if let Some(w) = &s.chat_wake_words { calls.wake_words = w.clone(); } let channels = s.chat_channels.iter() .map(|c| (c.id, ChatChannelCall { everything: c.answer_all, send: c.send })) .collect(); Self { calls, channels } } } /// Chat answers waited for longer than this are forgotten (the brain or the worker went away). const CHAT_ANSWER_KEPT: Duration = Duration::from_secs(300); /// The missed requests, shared between the conversation loop and the dashboard. pub type MissedLog = std::sync::Arc>>; /// A request that was not answered, or an answer that was stopped. No text: only who, why and when. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Missed { pub speaker: u64, pub reason: MissReason, pub at: Instant, pub chars: usize, } pub struct Conversation { voice: V, calls: CallSettings, policy: Policy, next_id: u64, /// Policy request id -> speech request id, and back. answers: HashMap, by_speech: HashMap, /// Ask id -> the request it becomes on a yes, and when it was asked. asks: HashMap, /// Shared with the dashboard (`missed_handle`). missed: MissedLog, chat: ChatCalls, /// Chat answer id -> the message it answers (channel, message) and when it was asked. chat_answers: HashMap, /// Each speaker's latest utterance that started and whose final has not come yet. started: HashMap, /// A wake word said alone, per speaker, waiting for what they say next; the deadline for the next /// utterance to start (None while it runs). wakes: HashMap)>, /// Requests whose earcon already played (a wake word said alone was answered with it). chimed: HashSet, } impl Conversation { pub fn new(voice: V, calls: CallSettings, policy: PolicySettings) -> Self { Self { voice, calls, policy: Policy::new(policy), next_id: 0, answers: HashMap::new(), by_speech: HashMap::new(), asks: HashMap::new(), missed: MissedLog::default(), chat: ChatCalls::default(), chat_answers: HashMap::new(), started: HashMap::new(), wakes: HashMap::new(), chimed: HashSet::new() } } /// Answer the registered chat channels too. pub fn with_chat(mut self, chat: ChatCalls) -> Self { self.chat = chat; self } fn on_chat(&mut self, m: &crate::chat::ChatMessage, now: Instant) { // Other bots are never answered, so two bots cannot keep answering each other. if m.author_is_bot || m.from_self { return; } let Some(ch) = self.chat.channels.get(&m.channel_id).copied() else { return }; if !ch.send { return; } let calls = CallSettings { mode: if ch.everything { crate::calls::CallMode::Everything } else { crate::calls::CallMode::WakeWord }, ..self.chat.calls.clone() }; let call = calls.judge(m.author_id, &m.content); debug_log!("conversation", "chat {} in {} from {} call={call:?}", m.message_id, m.channel_id, m.author_id); // A chat message is answered in chat; a stop there is not a question. if !matches!(call, Call::Request { .. }) { return; } self.chat_answers.retain(|_, (_, _, at)| now < *at + CHAT_ANSWER_KEPT); self.next_id += 1; let id = format!("c{}", self.next_id); if self.voice.chat_answer(&id, m.author_id, m.channel_id, &m.content) { self.chat_answers.insert(id, (m.channel_id, m.message_id, now)); } } /// A partial that already reads as a call (joined to a waiting wake word alone, as its final /// would be) lets the brain get ready while the speaker is still talking. Not while Jarvis is /// answering: the call may not be taken, and the brain is busy with the answer. fn on_partial(&mut self, speaker: u64, partial: &str) { if !self.answers.is_empty() { return; } let text = match self.wakes.get(&speaker) { Some((wake, _)) => format!("{} {}", wake.text, partial), None => partial.to_owned(), }; if matches!(self.calls.judge(speaker, &text), Call::Request { .. }) && !self.calls.wake_word_only(&text) { self.voice.prepare(speaker, &text); } } /// The missed requests kept, oldest first. pub fn missed(&self) -> std::vec::IntoIter { self.missed.lock().unwrap().iter().cloned().collect::>().into_iter() } /// The same records for the dashboard, readable while the loop runs. pub fn missed_handle(&self) -> MissedLog { self.missed.clone() } pub fn on_event(&mut self, ev: &CoreEvent, now: Instant) { match ev { CoreEvent::SpeechStart(h) => { self.policy.utterance_open(h.speaker_id); self.started.insert(h.speaker_id, h.utt); if let Some((_, deadline)) = self.wakes.get_mut(&h.speaker_id) { *deadline = None; // they go on: wait for this utterance's final } } CoreEvent::Chat(m) => self.on_chat(m, now), CoreEvent::Partial(h, partial) => self.on_partial(h.speaker_id, partial), CoreEvent::ReplyText { id, text, failed } => { if let Some((channel, message, asked)) = self.chat_answers.remove(id) { let late = now >= asked + CHAT_ANSWER_KEPT; debug_log!("conversation", "chat answer {id} chars={} failed={failed} late={late}", text.chars().count()); if !failed && !late && !text.trim().is_empty() { self.voice.send_reply(id, channel, message, text); } else { self.voice.reply_not_posted(id); } } } CoreEvent::SpeakingStopped(user) => self.policy.speaking_stopped(*user, now), CoreEvent::Final(f) => { let speaker = f.heard.speaker_id; if self.started.get(&speaker) == Some(&f.heard.utt) { self.started.remove(&speaker); } // What follows a wake word said alone is joined to it. let (text, chimed) = match self.wakes.remove(&speaker) { Some((wake, _)) => (format!("{} {}", wake.text, f.text), self.chimed.remove(&wake.id)), None => (f.text.clone(), false), }; self.next_id += 1; if chimed { self.chimed.insert(self.next_id); } let req = Request { id: self.next_id, speaker, text: text.clone(), direct: false, ended_at: now.checked_sub(Duration::from_millis(f.wait_ms)).unwrap_or(now), arrived_at: now }; let call = self.calls.judge(speaker, &text); debug_log!("conversation", "final speaker={speaker} req={} call={call:?}", req.id); if matches!(call, Call::Request { direct: true }) && self.calls.wake_word_only(&text) { let going_on = self.started.contains_key(&speaker); debug_log!("conversation", "wake word alone req={} speaker={speaker} going_on={going_on}", req.id); if !going_on { let a = self.policy.utterance_closed(speaker, now); self.apply(a, now); } // "I heard you" at once, as voice assistants do after their name; only when // Jarvis is free, since a busy Jarvis may not take the question. if self.calls.earcon && self.answers.is_empty() && self.chimed.insert(req.id) { self.voice.earcon(); } let deadline = (!going_on).then(|| now + self.calls.follow_up); self.wakes.insert(speaker, (Request { direct: true, ..req }, deadline)); return; } match call { Call::Ignore => { let a = self.policy.utterance_closed(speaker, now); self.apply(a, now); } Call::Stop => { self.voice.stop_all(); self.stop_everything(now); let a = self.policy.utterance_closed(speaker, now); self.apply(a, now); } Call::Request { direct } => { let a = self.policy.request(Request { direct, ..req }, now); self.apply(a, now); } Call::Ask => { let id = format!("q{}", req.id); if self.voice.ask_addressed(&id, speaker, &text) { self.asks.insert(id, (req, now)); } else { let a = self.policy.utterance_closed(speaker, now); self.apply(a, now); } } } } CoreEvent::Addressed { id, addressed } => { if let Some((req, asked)) = self.asks.remove(id) { // An answer after the timeout counts as no, even before the timer ran. let yes = *addressed && now < asked + ADDRESSED_TIMEOUT; debug_log!("conversation", "addressed req={} yes={yes}", req.id); let a = if yes { self.policy.request(req, now) } else { self.policy.utterance_closed(req.speaker, now) }; self.apply(a, now); } } CoreEvent::Ranked { id, order } => { let (Some(rank_id), order) = (id.strip_prefix('k').and_then(|n| n.parse().ok()), order.iter().filter_map(|o| o.parse().ok()).collect::>()) else { return }; let a = self.policy.ranked(rank_id, &order, now); self.apply(a, now); } CoreEvent::SpeakEnd(e) => { if let Some(pid) = self.by_speech.remove(&e.id) { self.answers.remove(&pid); debug_log!("conversation", "answer req={pid} done played={} cancelled={} failed={}", e.played, e.cancelled, e.failed); let a = self.policy.answer_done(pid, now); self.apply(a, now); } } // All speech was stopped (slash command, leaving): drop what was waiting before the stopped // speech's end could start the next answer. After a voice stop this finds nothing left. CoreEvent::Stopped => self.stop_everything(now), // Utterances in progress end with the session or the worker: their finals will not come // (a restarted worker numbers utterances afresh), so nothing waits for them. CoreEvent::Left { .. } => { self.started.clear(); self.stop_everything(now); } CoreEvent::WorkerLost => { self.started.clear(); for (_, (req, _)) in self.wakes.drain() { self.chimed.remove(&req.id); } } _ => {} } } /// The policy's stop, and every pending "was it said to Jarvis?" question is void. fn stop_everything(&mut self, now: Instant) { self.wakes.clear(); let a = self.policy.stop(); self.apply(a, now); self.chimed.clear(); // nothing is waiting any more for (_, (req, _)) in self.asks.drain().collect::>() { let a = self.policy.utterance_closed(req.speaker, now); self.apply(a, now); } } /// Timers: the policy's and unanswered "was it said to Jarvis?" questions. pub fn tick(&mut self, now: Instant) { let late: Vec = self.asks.iter() .filter(|(_, (_, at))| now >= *at + ADDRESSED_TIMEOUT).map(|(id, _)| id.clone()).collect(); for id in late { if let Some((req, _)) = self.asks.remove(&id) { debug_log!("conversation", "addressed req={} timed out", req.id); let a = self.policy.utterance_closed(req.speaker, now); self.apply(a, now); } } // Nothing followed a wake word said alone: it is the request. let due: Vec = self.wakes.iter() .filter(|(_, (_, d))| d.is_some_and(|d| now >= d)).map(|(s, _)| *s).collect(); for speaker in due { if let Some((req, _)) = self.wakes.remove(&speaker) { debug_log!("conversation", "wake word alone req={} answered alone", req.id); let a = self.policy.request(req, now); self.apply(a, now); } } let a = self.policy.tick(now); self.apply(a, now); } pub fn next_deadline(&self) -> Option { let asks = self.asks.values().map(|(_, at)| *at + ADDRESSED_TIMEOUT); let wakes = self.wakes.values().filter_map(|(_, d)| *d); self.policy.next_deadline().into_iter().chain(asks).chain(wakes).min() } fn apply(&mut self, actions: Vec, now: Instant) { for action in actions { match action { Action::Answer(r) => match { // The earcon plays first, then the answer as soon as it is written. if self.calls.earcon && !self.chimed.remove(&r.id) { self.voice.earcon(); } self.voice.answer(r.speaker, &r.text, (r.ended_at, r.arrived_at)) } { Some(sid) => { debug_log!("conversation", "answer req={} speaker={} speech={sid}", r.id, r.speaker); self.by_speech.insert(sid.clone(), r.id); self.answers.insert(r.id, sid); } None => { let a = self.policy.answer_done(r.id, now); self.apply(a, now); } }, Action::StopAnswer(pid) => { if let Some(sid) = self.answers.remove(&pid) { self.by_speech.remove(&sid); self.voice.cancel(&sid); } } Action::Missed(r, reason) => { self.chimed.remove(&r.id); debug_log!("conversation", "missed req={} speaker={} reason={reason:?}", r.id, r.speaker); let mut missed = self.missed.lock().unwrap(); if missed.len() == MISSED_KEPT { missed.pop_front(); } missed.push_back(Missed { speaker: r.speaker, reason, at: now, chars: r.text.chars().count() }); } Action::Rank { id, candidates } => { let c: Vec<(String, u64, String)> = candidates.iter().map(|r| (r.id.to_string(), r.speaker, r.text.clone())).collect(); debug_log!("conversation", "rank k{id} candidates={}", c.len()); // Not sent: the policy's ranking timeout picks the earliest-ended. self.voice.ask_rank(&format!("k{id}"), &c); } } } } } /// Drive a conversation from the core's events, passing every event on to `out`. pub async fn run(mut conv: Conversation, mut events: mpsc::UnboundedReceiver, out: mpsc::UnboundedSender) { loop { let deadline = conv.next_deadline(); tokio::select! { ev = events.recv() => match ev { Some(ev) => { conv.on_event(&ev, Instant::now()); let _ = out.send(ev); } None => return, }, _ = async { match deadline { Some(d) => tokio::time::sleep_until(d.into()).await, None => std::future::pending::<()>().await, } } => conv.tick(Instant::now()), } } } #[cfg(test)] mod tests { use super::*; use crate::discord::{Heard, HeardFinal}; use crate::playback::PlaybackEnd; use crate::policy::PolicyKind; use crate::calls::CallMode; use std::cell::RefCell; use std::rc::Rc; #[derive(Debug, Clone, PartialEq)] enum Did { Answer(u64, String), Cancel(String), StopAll, Earcon, Prepare(u64, String), Addressed(String, u64), Rank(String, Vec), ChatAnswer(String, u64, String), /// The channel a chat answer was asked for. ChatIn(u64), SendReply(String, u64, u64, String), NotPosted(String), } #[derive(Clone, Default)] struct Fake { did: Rc>>, n: Rc>, chat_in: Rc>>, } impl Voice for Fake { fn answer(&self, user: u64, text: &str, _heard: (Instant, Instant)) -> Option { *self.n.borrow_mut() += 1; self.did.borrow_mut().push(Did::Answer(user, text.into())); Some(format!("s{}", self.n.borrow())) } fn cancel(&self, id: &str) { self.did.borrow_mut().push(Did::Cancel(id.into())); } fn stop_all(&self) { self.did.borrow_mut().push(Did::StopAll); } fn earcon(&self) { self.did.borrow_mut().push(Did::Earcon); } fn prepare(&self, user: u64, text: &str) { self.did.borrow_mut().push(Did::Prepare(user, text.into())); } fn ask_addressed(&self, id: &str, user: u64, _text: &str) -> bool { self.did.borrow_mut().push(Did::Addressed(id.into(), user)); true } fn ask_rank(&self, id: &str, c: &[(String, u64, String)]) -> bool { self.did.borrow_mut().push(Did::Rank(id.into(), c.iter().map(|x| x.0.clone()).collect())); true } fn chat_answer(&self, id: &str, user: u64, channel: u64, text: &str) -> bool { self.chat_in.borrow_mut().push(Did::ChatIn(channel)); self.did.borrow_mut().push(Did::ChatAnswer(id.into(), user, text.into())); true } fn send_reply(&self, id: &str, channel: u64, message: u64, text: &str) { self.did.borrow_mut().push(Did::SendReply(id.into(), channel, message, text.into())); } fn reply_not_posted(&self, id: &str) { self.did.borrow_mut().push(Did::NotPosted(id.into())); } } impl Fake { fn take(&self) -> Vec { std::mem::take(&mut self.did.borrow_mut()) } } const A: u64 = 11; const B: u64 = 22; fn conv(kind: PolicyKind, mode: CallMode) -> (Conversation, Fake, Instant) { let f = Fake::default(); let calls = CallSettings { mode, earcon: false, ..CallSettings::default() }; let policy = PolicySettings { kind, collect: Duration::ZERO, ..PolicySettings::default() }; (Conversation::new(f.clone(), calls, policy), f, Instant::now()) } fn conv_earcon(kind: PolicyKind) -> (Conversation, Fake, Instant) { let f = Fake::default(); let policy = PolicySettings { kind, collect: Duration::ZERO, ..PolicySettings::default() }; (Conversation::new(f.clone(), CallSettings::default(), policy), f, Instant::now()) } #[test] fn the_earcon_plays_right_before_each_answer_is_asked_for() { let (mut c, f, t) = conv_earcon(PolicyKind::InOrder); c.on_event(&fin(A, "자비스 날씨 어때", 100), ms(t, 0)); assert_eq!(f.take(), vec![Did::Earcon, Did::Answer(A, "자비스 날씨 어때".into())]); c.on_event(&fin(B, "자비스 불 꺼 줘", 100), ms(t, 200)); assert!(f.take().is_empty(), "a waiting request gets its earcon when its answer starts"); c.on_event(&end("s1", false), ms(t, 3000)); assert_eq!(f.take(), vec![Did::Earcon, Did::Answer(B, "자비스 불 꺼 줘".into())]); c.on_event(&fin(B, "오늘 뭐 먹지", 100), ms(t, 3100)); assert!(f.take().is_empty(), "no call, no earcon"); } #[test] fn a_wake_word_alone_gets_the_earcon_at_once_and_its_question_does_not_get_another() { let (mut c, f, t) = conv_earcon(PolicyKind::First); c.on_event(&start(A, 1), ms(t, 0)); c.on_event(&fin_utt(A, 1, "자비스"), ms(t, 900)); assert_eq!(f.take(), vec![Did::Earcon]); c.on_event(&start(A, 2), ms(t, 1200)); c.on_event(&fin_utt(A, 2, "지금 몇 시야?"), ms(t, 3000)); assert_eq!(f.take(), vec![Did::Answer(A, "자비스 지금 몇 시야?".into())]); // Nothing followed the next one: it is answered alone, still with one earcon. c.on_event(&end("s1", false), ms(t, 5000)); c.on_event(&fin_utt(A, 3, "자비스"), ms(t, 6000)); c.tick(ms(t, 6000) + CallSettings::default().follow_up); assert_eq!(f.take(), vec![Did::Earcon, Did::Answer(A, "자비스".into())]); } #[test] fn a_busy_jarvis_gives_no_earcon_for_a_call_it_does_not_take() { let (mut c, f, t) = conv_earcon(PolicyKind::First); c.on_event(&fin(A, "자비스 날씨", 100), ms(t, 0)); assert_eq!(f.take(), vec![Did::Earcon, Did::Answer(A, "자비스 날씨".into())]); c.on_event(&fin(B, "자비스 안녕", 100), ms(t, 100)); c.on_event(&fin_utt(B, 2, "자비스"), ms(t, 200)); assert!(f.take().is_empty(), "missed while busy, and a wake word alone waits silently"); } fn partial(speaker: u64, text: &str) -> CoreEvent { CoreEvent::Partial(Heard { speaker_id: speaker, speaker_name: "x".into(), utt: 1, session: Some(1) }, text.into()) } #[test] fn a_partial_that_reads_as_a_call_lets_the_brain_get_ready() { let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord); c.on_event(&partial(A, "자비스 오늘"), ms(t, 0)); c.on_event(&partial(B, "오늘 뭐 먹지"), ms(t, 0)); // not a call c.on_event(&partial(A, "자비스"), ms(t, 0)); // the name alone: nothing to get ready for c.on_event(&partial(A, "자비스 멈춰"), ms(t, 0)); // a stop is not answered assert_eq!(f.take(), vec![Did::Prepare(A, "자비스 오늘".into())]); // After a wake word said alone, the next utterance's partial is joined to it, as its final. c.on_event(&fin_utt(B, 1, "자비스"), ms(t, 100)); c.on_event(&partial(B, "지금 몇 시"), ms(t, 300)); assert_eq!(f.take(), vec![Did::Prepare(B, "자비스 지금 몇 시".into())]); // Not while an answer runs. c.on_event(&fin(A, "자비스 날씨", 100), ms(t, 400)); f.take(); c.on_event(&partial(A, "자비스 내일은"), ms(t, 500)); assert!(f.take().is_empty()); } #[test] fn the_earcon_can_be_turned_off() { let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord); c.on_event(&fin_utt(A, 1, "자비스"), ms(t, 0)); c.on_event(&fin_utt(A, 2, "안녕"), ms(t, 500)); assert_eq!(f.take(), vec![Did::Answer(A, "자비스 안녕".into())]); } fn fin(speaker: u64, text: &str, wait_ms: u64) -> CoreEvent { CoreEvent::Final(HeardFinal { heard: Heard { speaker_id: speaker, speaker_name: "x".into(), utt: 1, session: Some(1) }, text: text.into(), end_reason: "silence".into(), audio_ms: 1000, wait_ms, stt_ms: 100 }) } fn end(id: &str, cancelled: bool) -> CoreEvent { CoreEvent::SpeakEnd(PlaybackEnd { id: id.into(), played: 1, cancelled, failed: false }) } fn ms(t: Instant, n: u64) -> Instant { t + Duration::from_millis(n) } fn fin_utt(speaker: u64, utt: u32, text: &str) -> CoreEvent { CoreEvent::Final(HeardFinal { heard: Heard { speaker_id: speaker, speaker_name: "x".into(), utt, session: Some(1) }, text: text.into(), end_reason: "silence".into(), audio_ms: 1000, wait_ms: 100, stt_ms: 100 }) } fn start(speaker: u64, utt: u32) -> CoreEvent { CoreEvent::SpeechStart(Heard { speaker_id: speaker, speaker_name: "x".into(), utt, session: Some(1) }) } #[test] fn a_wake_word_alone_waits_for_what_the_speaker_says_next() { let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord); // "자비스," then a pause that ended the utterance; the question follows. c.on_event(&start(A, 1), ms(t, 0)); c.on_event(&fin_utt(A, 1, "자비스,"), ms(t, 900)); assert!(f.take().is_empty(), "a wake word alone is not the question yet"); c.on_event(&start(A, 2), ms(t, 1200)); c.tick(ms(t, 5000)); // still talking: no time limit while the next utterance runs assert!(f.take().is_empty()); c.on_event(&fin_utt(A, 2, "지금 몇 시야?"), ms(t, 5200)); assert_eq!(f.take(), vec![Did::Answer(A, "자비스, 지금 몇 시야?".into())]); assert_eq!(c.missed().count(), 0); } #[test] fn a_wake_word_alone_waits_for_a_next_utterance_that_already_started() { let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord); c.on_event(&start(A, 1), ms(t, 0)); c.on_event(&start(A, 2), ms(t, 600)); // the final of utterance 1 comes after 2 started c.on_event(&fin_utt(A, 1, "자비스"), ms(t, 900)); c.tick(ms(t, 900) + CallSettings::default().follow_up + Duration::from_millis(1)); assert!(f.take().is_empty()); c.on_event(&fin_utt(A, 2, "멈춰"), ms(t, 3000)); assert_eq!(f.take(), vec![Did::StopAll], "joined, it is a stop"); } #[test] fn a_wake_word_alone_is_answered_when_nothing_follows() { let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord); c.on_event(&start(A, 1), ms(t, 0)); c.on_event(&fin_utt(A, 1, "자비스"), ms(t, 900)); let wait = CallSettings::default().follow_up; assert_eq!(c.next_deadline(), Some(ms(t, 900) + wait)); c.tick(ms(t, 900) + wait - Duration::from_millis(1)); assert!(f.take().is_empty()); c.tick(ms(t, 900) + wait); assert_eq!(f.take(), vec![Did::Answer(A, "자비스".into())]); // Another speaker's words do not join it. c.on_event(&end("s1", false), ms(t, 4000)); c.on_event(&start(A, 2), ms(t, 4100)); c.on_event(&fin_utt(A, 2, "자비스"), ms(t, 4500)); c.on_event(&fin_utt(B, 1, "오늘 뭐 먹지"), ms(t, 4600)); c.tick(ms(t, 4500) + wait); assert_eq!(f.take(), vec![Did::Answer(A, "자비스".into())]); } #[test] fn a_lost_worker_drops_a_waiting_wake_word_and_what_was_running() { let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord); c.on_event(&start(A, 1), ms(t, 0)); c.on_event(&fin_utt(A, 1, "자비스"), ms(t, 900)); c.on_event(&start(A, 2), ms(t, 1200)); // the final of utterance 2 is lost with the worker c.on_event(&CoreEvent::WorkerLost, ms(t, 2000)); assert_eq!(c.next_deadline(), None); // The restarted worker numbers utterances from 1 again; nothing old joins or holds them. c.on_event(&start(A, 1), ms(t, 60_000)); c.on_event(&fin_utt(A, 1, "오늘 뭐 먹지"), ms(t, 61_000)); assert!(f.take().is_empty(), "no wake word: not a call"); c.on_event(&start(A, 2), ms(t, 62_000)); c.on_event(&fin_utt(A, 2, "자비스"), ms(t, 63_000)); let wait = CallSettings::default().follow_up; assert_eq!(c.next_deadline(), Some(ms(t, 63_000) + wait)); c.tick(ms(t, 63_000) + wait); assert_eq!(f.take(), vec![Did::Answer(A, "자비스".into())]); } #[test] fn a_stop_drops_a_waiting_wake_word() { let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord); c.on_event(&fin_utt(A, 1, "자비스"), ms(t, 900)); c.on_event(&CoreEvent::Stopped, ms(t, 1000)); c.tick(ms(t, 900) + CallSettings::default().follow_up); assert!(f.take().is_empty()); assert_eq!(c.next_deadline(), None); } #[test] fn a_call_is_answered_and_the_next_one_waits_for_its_speech_to_end() { let (mut c, f, t) = conv(PolicyKind::InOrder, CallMode::WakeWord); c.on_event(&fin(A, "자비스 날씨 어때", 100), ms(t, 0)); assert_eq!(f.take(), vec![Did::Answer(A, "자비스 날씨 어때".into())]); c.on_event(&fin(B, "오늘 뭐 먹지", 100), ms(t, 100)); // not a call c.on_event(&fin(B, "자비스 불 꺼 줘", 100), ms(t, 200)); assert!(f.take().is_empty(), "queued behind the answer"); c.on_event(&end("s1", false), ms(t, 3000)); assert_eq!(f.take(), vec![Did::Answer(B, "자비스 불 꺼 줘".into())]); assert_eq!(c.missed().count(), 0); } #[test] fn first_drops_a_call_while_busy_and_records_it_without_text() { let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord); c.on_event(&fin(A, "자비스 날씨", 100), ms(t, 0)); c.on_event(&fin(B, "자비스 안녕", 100), ms(t, 100)); assert_eq!(f.take().len(), 1); let m: Vec<_> = c.missed().collect(); assert_eq!(m, vec![Missed { speaker: B, reason: MissReason::Busy, at: ms(t, 100), chars: "자비스 안녕".chars().count() }]); } #[test] fn a_voice_stop_silences_everything_and_drops_the_queue() { let (mut c, f, t) = conv(PolicyKind::InOrder, CallMode::WakeWord); c.on_event(&fin(A, "자비스 날씨", 100), ms(t, 0)); c.on_event(&fin(B, "자비스 안녕", 100), ms(t, 100)); f.take(); c.on_event(&fin(B, "자비스 멈춰", 100), ms(t, 200)); let did = f.take(); assert_eq!(did[0], Did::StopAll); assert!(!did.iter().any(|d| matches!(d, Did::Answer(..)))); // The stopped speech's end starts nothing. c.on_event(&end("s1", true), ms(t, 300)); assert!(f.take().iter().all(|d| !matches!(d, Did::Answer(..)))); assert_eq!(c.missed().filter(|m| m.reason == MissReason::Stopped).count(), 2); } #[test] fn addressed_mode_asks_the_brain_and_answers_only_a_yes() { let (mut c, f, t) = conv(PolicyKind::InOrder, CallMode::Addressed); c.on_event(&fin(A, "불 좀 꺼 줄래", 100), ms(t, 0)); let Did::Addressed(q, user) = f.take().remove(0) else { panic!() }; assert_eq!(user, A); c.on_event(&CoreEvent::Addressed { id: q, addressed: true }, ms(t, 300)); assert_eq!(f.take(), vec![Did::Answer(A, "불 좀 꺼 줄래".into())]); c.on_event(&end("s1", false), ms(t, 2000)); c.on_event(&fin(B, "너 어제 그거 봤어?", 100), ms(t, 2100)); let Did::Addressed(q, _) = f.take().remove(0) else { panic!() }; c.on_event(&CoreEvent::Addressed { id: q, addressed: false }, ms(t, 2300)); assert!(f.take().is_empty()); // No answer at all counts as no once the timeout passes. c.on_event(&fin(B, "음", 100), ms(t, 3000)); f.take(); assert_eq!(c.next_deadline(), Some(ms(t, 3000) + ADDRESSED_TIMEOUT)); c.tick(ms(t, 3000) + ADDRESSED_TIMEOUT); assert!(f.take().is_empty()); assert_eq!(c.next_deadline(), None); } #[test] fn latest_cancels_the_running_answer_for_a_newer_call() { let (mut c, f, t) = conv(PolicyKind::Latest, CallMode::WakeWord); c.on_event(&fin(A, "자비스 날씨", 100), ms(t, 0)); c.on_event(&fin(B, "자비스 뉴스", 100), ms(t, 500)); assert_eq!(f.take(), vec![Did::Answer(A, "자비스 날씨".into()), Did::Cancel("s1".into()), Did::Answer(B, "자비스 뉴스".into())]); // The cancelled speech's end does not end the new answer. c.on_event(&end("s1", true), ms(t, 600)); c.on_event(&fin(A, "자비스 하나 더", 100), ms(t, 700)); assert_eq!(f.take(), vec![Did::Cancel("s2".into()), Did::Answer(A, "자비스 하나 더".into())]); } #[test] fn a_tie_in_important_is_ranked_by_the_brain() { let (mut c, f, t) = conv(PolicyKind::Important, CallMode::WakeWord); c.on_event(&fin(A, "자비스 날씨", 100), ms(t, 0)); f.take(); c.on_event(&fin(B, "자비스 불 꺼", 100), ms(t, 200)); let Did::Rank(k, ids) = f.take().remove(0) else { panic!() }; assert_eq!(ids, vec!["1", "2"]); c.on_event(&CoreEvent::Ranked { id: k, order: vec!["2".into(), "1".into()] }, ms(t, 400)); assert_eq!(f.take(), vec![Did::Cancel("s1".into()), Did::Answer(B, "자비스 불 꺼".into())]); } #[test] fn a_voice_stop_also_drops_a_pending_addressed_question() { let (mut c, f, t) = conv(PolicyKind::InOrder, CallMode::Addressed); c.on_event(&fin(A, "불 좀 꺼 줄래", 100), ms(t, 0)); let Did::Addressed(q, _) = f.take().remove(0) else { panic!() }; c.on_event(&fin(B, "자비스 멈춰", 100), ms(t, 100)); f.take(); c.on_event(&CoreEvent::Addressed { id: q, addressed: true }, ms(t, 300)); assert!(f.take().is_empty(), "the question asked before the stop is void"); assert_eq!(c.next_deadline(), None); } #[test] fn a_stop_from_elsewhere_drops_the_queue_before_the_cancelled_speech_ends() { let (mut c, f, t) = conv(PolicyKind::InOrder, CallMode::Addressed); c.on_event(&fin(A, "자비스 날씨", 100), ms(t, 0)); c.on_event(&fin(B, "자비스 뉴스", 100), ms(t, 100)); c.on_event(&fin(B, "불 좀 꺼 줄래", 100), ms(t, 200)); // waiting for "addressed" let did = f.take(); let q = did.iter().find_map(|d| if let Did::Addressed(q, _) = d { Some(q.clone()) } else { None }).unwrap(); // The slash command stops all speech: Stopped comes before the speech's end. c.on_event(&CoreEvent::Stopped, ms(t, 300)); c.on_event(&end("s1", true), ms(t, 310)); c.on_event(&CoreEvent::Addressed { id: q, addressed: true }, ms(t, 400)); assert!(!f.take().iter().any(|d| matches!(d, Did::Answer(..))), "nothing starts after a stop"); assert_eq!(c.missed().filter(|m| m.reason == MissReason::Stopped).count(), 2); } #[test] fn an_addressed_answer_after_its_timeout_counts_as_no() { let (mut c, f, t) = conv(PolicyKind::InOrder, CallMode::Addressed); c.on_event(&fin(A, "불 좀 꺼 줄래", 100), ms(t, 0)); let Did::Addressed(q, _) = f.take().remove(0) else { panic!() }; c.on_event(&CoreEvent::Addressed { id: q, addressed: true }, ms(t, 0) + ADDRESSED_TIMEOUT); assert!(f.take().is_empty()); } const CH_WAKE: u64 = 100; const CH_ALL: u64 = 200; const CH_MUTE: u64 = 300; fn chat_conv(calls: CallSettings) -> (Conversation, Fake, Instant) { let f = Fake::default(); let chat = ChatCalls { calls, channels: [ (CH_WAKE, ChatChannelCall { everything: false, send: true }), (CH_ALL, ChatChannelCall { everything: true, send: true }), (CH_MUTE, ChatChannelCall { everything: true, send: false }), ].into() }; let voice = CallSettings { earcon: false, ..CallSettings::default() }; let c = Conversation::new(f.clone(), voice, PolicySettings::default()).with_chat(chat); (c, f, Instant::now()) } fn chat(channel: u64, message: u64, author: u64, bot: bool, text: &str) -> CoreEvent { CoreEvent::Chat(crate::chat::ChatMessage { channel_id: channel, message_id: message, author_id: author, author_name: "x".into(), author_is_bot: bot, from_self: false, content: text.into(), created_at: String::new() }) } #[test] fn a_chat_call_is_answered_as_a_reply_to_its_message() { let (mut c, f, t) = chat_conv(CallSettings::default()); c.on_event(&chat(CH_WAKE, 5, A, false, "자비스 내일 일정 알려 줘"), t); let Did::ChatAnswer(id, user, text) = f.take().remove(0) else { panic!() }; assert_eq!((user, text.as_str()), (A, "자비스 내일 일정 알려 줘")); assert_eq!(*f.chat_in.borrow(), vec![Did::ChatIn(CH_WAKE)], "the brain is told where it was asked"); c.on_event(&CoreEvent::ReplyText { id: id.clone(), text: "내일은 비어 있어요.".into(), failed: false }, t); assert_eq!(f.take(), vec![Did::SendReply(id.clone(), CH_WAKE, 5, "내일은 비어 있어요.".into())]); // A second reply_text for the same id sends nothing. c.on_event(&CoreEvent::ReplyText { id, text: "x".into(), failed: false }, t); assert!(f.take().is_empty()); } #[test] fn chat_answers_follow_each_channels_switch_and_never_answer_bots() { let (mut c, f, t) = chat_conv(CallSettings::default()); c.on_event(&chat(CH_WAKE, 1, A, false, "오늘 뭐 먹지"), t); assert!(f.take().is_empty(), "wake word only"); c.on_event(&chat(CH_ALL, 2, A, false, "오늘 뭐 먹지"), t); assert!(matches!(f.take()[..], [Did::ChatAnswer(..)]), "every message"); c.on_event(&chat(CH_MUTE, 3, A, false, "자비스 안녕"), t); assert!(f.take().is_empty(), "sending is off: no answer could be posted"); c.on_event(&chat(CH_ALL, 4, B, true, "자비스 안녕"), t); assert!(f.take().is_empty(), "other bots are never answered"); c.on_event(&chat(999, 6, A, false, "자비스 안녕"), t); assert!(f.take().is_empty(), "not a registered channel"); c.on_event(&chat(CH_ALL, 7, A, false, "자비스 멈춰"), t); assert!(f.take().is_empty(), "a stop in chat is not a question"); } #[test] fn chat_uses_its_own_wake_words_and_the_designated_people() { let calls = CallSettings { wake_words: vec!["봇".into()], designated: [A].into(), designated_only: true, ..CallSettings::default() }; let (mut c, f, t) = chat_conv(calls); c.on_event(&chat(CH_WAKE, 1, A, false, "봇 안녕"), t); assert_eq!(f.take().len(), 1); c.on_event(&chat(CH_WAKE, 2, B, false, "봇 안녕"), t); assert!(f.take().is_empty(), "only designated people"); c.on_event(&chat(CH_WAKE, 3, A, false, "자비스 안녕"), t); assert!(f.take().is_empty(), "the voice wake word is not the chat one here"); } #[test] fn a_failed_chat_answer_posts_nothing_and_chat_leaves_voice_alone() { let (mut c, f, t) = chat_conv(CallSettings::default()); c.on_event(&fin(A, "자비스 날씨", 100), t); c.on_event(&chat(CH_ALL, 9, B, false, "안녕"), t); let did = f.take(); assert!(matches!(did[0], Did::Answer(..)) && matches!(did[1], Did::ChatAnswer(..)), "{did:?}"); let Did::ChatAnswer(id, ..) = &did[1] else { panic!() }; c.on_event(&CoreEvent::ReplyText { id: id.clone(), text: String::new(), failed: true }, t); assert_eq!(f.take(), vec![Did::NotPosted(id.clone())]); assert_eq!(c.missed().count(), 0, "a chat message never takes the voice turn"); } #[test] fn a_chat_answer_after_its_keep_time_is_not_posted() { let (mut c, f, t) = chat_conv(CallSettings::default()); c.on_event(&chat(CH_WAKE, 5, A, false, "자비스 안녕"), t); let Did::ChatAnswer(id, ..) = f.take().remove(0) else { panic!() }; c.on_event(&CoreEvent::ReplyText { id: id.clone(), text: "안녕하세요.".into(), failed: false }, t + CHAT_ANSWER_KEPT); assert_eq!(f.take(), vec![Did::NotPosted(id)], "too late: the conversation has moved on"); } #[test] fn end_times_come_from_the_finals_wait() { let (mut c, f, t) = conv(PolicyKind::InOrder, CallMode::WakeWord); c.on_event(&fin(A, "자비스 하나", 0), ms(t, 0)); c.on_event(&fin(B, "자비스 둘", 0), ms(t, 1000)); c.on_event(&fin(A, "자비스 셋", 900), ms(t, 1500)); // ended at 600, before 둘 f.take(); c.on_event(&end("s1", false), ms(t, 2000)); assert_eq!(f.take(), vec![Did::Answer(A, "자비스 셋".into())]); } }