From bd2eb49e3d512adf1f8bfddacd47d2579f5c3cd0 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Thu, 12 Nov 2026 09:44:06 +0900 Subject: [PATCH] fix(app): a wake word said alone waits for what the speaker says next MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pause after the name ("자비스, ... 지금 몇 시야?") is longer than the worker's 300 ms end silence, so the wake word ended its own utterance: Jarvis answered the bare name and ignored the question that followed (no wake word). Found on real Discord. A voice utterance that is only a wake word is now joined to the same speaker's next final when that utterance is running or starts within call.followUpMs (default 1500); otherwise the wake word alone is the request after that time. Stops drop it. Co-Authored-By: Claude Opus 5.5 --- app/src/calls.rs | 20 ++++++ app/src/conversation.rs | 121 ++++++++++++++++++++++++++++++++--- app/src/conversation.spec.md | 12 ++++ app/src/settings.rs | 6 +- docs/v2/conversation.md | 1 + 5 files changed, 149 insertions(+), 11 deletions(-) diff --git a/app/src/calls.rs b/app/src/calls.rs index fdf7148..92d8667 100644 --- a/app/src/calls.rs +++ b/app/src/calls.rs @@ -4,6 +4,7 @@ //! matching works on Unicode-normalised letters, so it behaves the same in any script. use std::collections::HashSet; +use std::time::Duration; use unicode_normalization::UnicodeNormalization; @@ -27,6 +28,9 @@ pub struct CallSettings { pub designated_only: bool, pub designated_no_wake_word: bool, pub stop_words: Vec, + /// After a wake word said alone, how long the speaker's next utterance may take to start and + /// still be joined to it (the pause after "Jarvis," often ends an utterance). + pub follow_up: Duration, } impl Default for CallSettings { @@ -39,6 +43,7 @@ impl Default for CallSettings { designated_only: false, designated_no_wake_word: false, stop_words: ["멈춰", "그만", "정지", "stop"].map(String::from).to_vec(), + follow_up: Duration::from_millis(1500), } } } @@ -122,6 +127,11 @@ impl CallSettings { }) } + /// A wake word is heard and nothing else (no letters or digits around it). + pub fn wake_word_only(&self, text: &str) -> bool { + self.without_wake_word(&normalise(text)).is_some_and(|rest| rest.is_empty()) + } + /// Decide what to do with one finished utterance. pub fn judge(&self, speaker: u64, text: &str) -> Call { let norm = normalise(text); @@ -160,6 +170,16 @@ mod tests { CallSettings::default() } + #[test] + fn a_wake_word_with_nothing_else_is_told_apart() { + let c = s(); + assert!(c.wake_word_only("자비스,")); + assert!(c.wake_word_only("자비즈?")); + assert!(!c.wake_word_only("자비스, 지금 몇 시야?")); + assert!(!c.wake_word_only("서비스")); + assert!(!c.wake_word_only("")); + } + #[test] fn exact_and_near_wake_words_call_but_a_different_word_does_not() { let c = s(); diff --git a/app/src/conversation.rs b/app/src/conversation.rs index 5ab090a..2c4015b 100644 --- a/app/src/conversation.rs +++ b/app/src/conversation.rs @@ -130,13 +130,18 @@ pub struct Conversation { chat: ChatCalls, /// Chat answer id -> the message it answers (channel, message) and when it was asked. chat_answers: HashMap, + /// Each speaker's latest utterance that started. + started: HashMap, + /// A wake word said alone, per speaker, waiting for what they say next; the deadline for the next + /// utterance to start (None while it runs). + wakes: HashMap)>, } impl Conversation { pub fn new(voice: V, calls: CallSettings, policy: PolicySettings) -> Self { Self { voice, calls, policy: Policy::new(policy), next_id: 0, answers: HashMap::new(), by_speech: HashMap::new(), asks: HashMap::new(), missed: VecDeque::new(), chat: ChatCalls::default(), - chat_answers: HashMap::new() } + chat_answers: HashMap::new(), started: HashMap::new(), wakes: HashMap::new() } } /// Answer the registered chat channels too. @@ -178,7 +183,13 @@ impl Conversation { pub fn on_event(&mut self, ev: &CoreEvent, now: Instant) { match ev { - CoreEvent::SpeechStart(h) => self.policy.utterance_open(h.speaker_id), + CoreEvent::SpeechStart(h) => { + self.policy.utterance_open(h.speaker_id); + self.started.insert(h.speaker_id, h.utt); + if let Some((_, deadline)) = self.wakes.get_mut(&h.speaker_id) { + *deadline = None; // they go on: wait for this utterance's final + } + } CoreEvent::Chat(m) => self.on_chat(m, now), CoreEvent::ReplyText { id, text, failed } => { if let Some((channel, message, asked)) = self.chat_answers.remove(id) { @@ -195,11 +206,27 @@ impl Conversation { CoreEvent::SpeakingStopped(user) => self.policy.speaking_stopped(*user, now), CoreEvent::Final(f) => { let speaker = f.heard.speaker_id; + // What follows a wake word said alone is joined to it. + let text = match self.wakes.remove(&speaker) { + Some((wake, _)) => format!("{} {}", wake.text, f.text), + None => f.text.clone(), + }; self.next_id += 1; - let req = Request { id: self.next_id, speaker, text: f.text.clone(), direct: false, + let req = Request { id: self.next_id, speaker, text: text.clone(), direct: false, ended_at: now.checked_sub(Duration::from_millis(f.wait_ms)).unwrap_or(now), arrived_at: now }; - let call = self.calls.judge(speaker, &f.text); + let call = self.calls.judge(speaker, &text); debug_log!("conversation", "final speaker={speaker} req={} call={call:?}", req.id); + if matches!(call, Call::Request { direct: true }) && self.calls.wake_word_only(&text) { + let going_on = self.started.get(&speaker).is_some_and(|&u| u > f.heard.utt); + debug_log!("conversation", "wake word alone req={} speaker={speaker} going_on={going_on}", req.id); + if !going_on { + let a = self.policy.utterance_closed(speaker, now); + self.apply(a, now); + } + let deadline = (!going_on).then(|| now + self.calls.follow_up); + self.wakes.insert(speaker, (Request { direct: true, ..req }, deadline)); + return; + } match call { Call::Ignore => { let a = self.policy.utterance_closed(speaker, now); @@ -217,7 +244,7 @@ impl Conversation { } Call::Ask => { let id = format!("q{}", req.id); - if self.voice.ask_addressed(&id, speaker, &f.text) { + if self.voice.ask_addressed(&id, speaker, &text) { self.asks.insert(id, (req, now)); } else { let a = self.policy.utterance_closed(speaker, now); @@ -263,6 +290,7 @@ impl Conversation { /// The policy's stop, and every pending "was it said to Jarvis?" question is void. fn stop_everything(&mut self, now: Instant) { + self.wakes.clear(); let a = self.policy.stop(); self.apply(a, now); for (_, (req, _)) in self.asks.drain().collect::>() { @@ -282,16 +310,24 @@ impl Conversation { self.apply(a, now); } } + // Nothing followed a wake word said alone: it is the request. + let due: Vec = self.wakes.iter() + .filter(|(_, (_, d))| d.is_some_and(|d| now >= d)).map(|(s, _)| *s).collect(); + for speaker in due { + if let Some((req, _)) = self.wakes.remove(&speaker) { + debug_log!("conversation", "wake word alone req={} answered alone", req.id); + let a = self.policy.request(req, now); + self.apply(a, now); + } + } let a = self.policy.tick(now); self.apply(a, now); } pub fn next_deadline(&self) -> Option { - let asks = self.asks.values().map(|(_, at)| *at + ADDRESSED_TIMEOUT).min(); - match (self.policy.next_deadline(), asks) { - (Some(a), Some(b)) => Some(a.min(b)), - (a, b) => a.or(b), - } + let asks = self.asks.values().map(|(_, at)| *at + ADDRESSED_TIMEOUT); + let wakes = self.wakes.values().filter_map(|(_, d)| *d); + self.policy.next_deadline().into_iter().chain(asks).chain(wakes).min() } fn apply(&mut self, actions: Vec, now: Instant) { @@ -440,6 +476,71 @@ mod tests { t + Duration::from_millis(n) } + fn fin_utt(speaker: u64, utt: u32, text: &str) -> CoreEvent { + CoreEvent::Final(HeardFinal { heard: Heard { speaker_id: speaker, speaker_name: "x".into(), utt }, + text: text.into(), end_reason: "silence".into(), audio_ms: 1000, wait_ms: 100, stt_ms: 100 }) + } + fn start(speaker: u64, utt: u32) -> CoreEvent { + CoreEvent::SpeechStart(Heard { speaker_id: speaker, speaker_name: "x".into(), utt }) + } + + #[test] + fn a_wake_word_alone_waits_for_what_the_speaker_says_next() { + let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord); + // "자비스," then a pause that ended the utterance; the question follows. + c.on_event(&start(A, 1), ms(t, 0)); + c.on_event(&fin_utt(A, 1, "자비스,"), ms(t, 900)); + assert!(f.take().is_empty(), "a wake word alone is not the question yet"); + c.on_event(&start(A, 2), ms(t, 1200)); + c.tick(ms(t, 5000)); // still talking: no time limit while the next utterance runs + assert!(f.take().is_empty()); + c.on_event(&fin_utt(A, 2, "지금 몇 시야?"), ms(t, 5200)); + assert_eq!(f.take(), vec![Did::Answer(A, "자비스, 지금 몇 시야?".into())]); + assert_eq!(c.missed().count(), 0); + } + + #[test] + fn a_wake_word_alone_waits_for_a_next_utterance_that_already_started() { + let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord); + c.on_event(&start(A, 1), ms(t, 0)); + c.on_event(&start(A, 2), ms(t, 600)); // the final of utterance 1 comes after 2 started + c.on_event(&fin_utt(A, 1, "자비스"), ms(t, 900)); + c.tick(ms(t, 900) + CallSettings::default().follow_up + Duration::from_millis(1)); + assert!(f.take().is_empty()); + c.on_event(&fin_utt(A, 2, "멈춰"), ms(t, 3000)); + assert_eq!(f.take(), vec![Did::StopAll], "joined, it is a stop"); + } + + #[test] + fn a_wake_word_alone_is_answered_when_nothing_follows() { + let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord); + c.on_event(&start(A, 1), ms(t, 0)); + c.on_event(&fin_utt(A, 1, "자비스"), ms(t, 900)); + let wait = CallSettings::default().follow_up; + assert_eq!(c.next_deadline(), Some(ms(t, 900) + wait)); + c.tick(ms(t, 900) + wait - Duration::from_millis(1)); + assert!(f.take().is_empty()); + c.tick(ms(t, 900) + wait); + assert_eq!(f.take(), vec![Did::Answer(A, "자비스".into())]); + // Another speaker's words do not join it. + c.on_event(&end("s1", false), ms(t, 4000)); + c.on_event(&start(A, 2), ms(t, 4100)); + c.on_event(&fin_utt(A, 2, "자비스"), ms(t, 4500)); + c.on_event(&fin_utt(B, 1, "오늘 뭐 먹지"), ms(t, 4600)); + c.tick(ms(t, 4500) + wait); + assert_eq!(f.take(), vec![Did::Answer(A, "자비스".into())]); + } + + #[test] + fn a_stop_drops_a_waiting_wake_word() { + let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord); + c.on_event(&fin_utt(A, 1, "자비스"), ms(t, 900)); + c.on_event(&CoreEvent::Stopped, ms(t, 1000)); + c.tick(ms(t, 900) + CallSettings::default().follow_up); + assert!(f.take().is_empty()); + assert_eq!(c.next_deadline(), None); + } + #[test] fn a_call_is_answered_and_the_next_one_waits_for_its_speech_to_end() { let (mut c, f, t) = conv(PolicyKind::InOrder, CallMode::WakeWord); diff --git a/app/src/conversation.spec.md b/app/src/conversation.spec.md index 4801eaf..90e3492 100644 --- a/app/src/conversation.spec.md +++ b/app/src/conversation.spec.md @@ -32,6 +32,7 @@ Settings (`settings.json`, section `call`; wake words stay in `voice.wakeWords`) | `call.designatedOnly` | `false` | Answer only designated people. | | `call.designatedNoWakeWord` | `false` | Answer designated people without a wake word. | | `call.stopWords` | `["멈춰", "그만", "정지", "stop"]` | Said together with a wake word, they stop Jarvis in any policy. | +| `call.followUpMs` | `1500` | After a wake word said alone, how long the speaker's next utterance may take to start and still be joined to it. 0-5000. | Normalisation: Unicode NFKD, lower case, and only letters and digits kept (spaces, punctuation and combining marks dropped). NFKD splits Hangul syllables into their letters, so "자비즈" is one letter @@ -54,6 +55,17 @@ Decision for one utterance (`judge(speaker, text)`), in order: The text sent to the brain is the utterance as heard; the brain treats the wake word as address. +**A wake word said alone.** People often pause after the name ("자비스, ... 지금 몇 시야?"), and a +pause longer than the worker's `end_silence_ms` ends the utterance there. A voice utterance that is a +wake word and nothing else (no letters or digits left once the wake word is removed) is therefore +not a request yet: the conversation keeps it for that speaker and joins it to their next final, with +a space between, and judges the joined text as one utterance (so "자비스" then "멈춰" is a stop). If +the speaker's next utterance already started when the wake word's final came, or starts within +`call.followUpMs` of it, the wake word waits for that utterance's final however long it takes; +otherwise, after `call.followUpMs`, the wake word alone is the request. Other speakers' utterances +never join it, and a stop (voice, slash command or leaving) drops it. Chat messages are not joined: +a message is complete as sent. + ## Response policy Settings (section `response`): diff --git a/app/src/settings.rs b/app/src/settings.rs index d14e37d..b413f6c 100644 --- a/app/src/settings.rs +++ b/app/src/settings.rs @@ -242,6 +242,7 @@ fn parse_call(raw: &Map, wake_words: &[String], d: &CallSettings) designated_only: get_bool(call, "designatedOnly", "call", d.designated_only)?, designated_no_wake_word: get_bool(call, "designatedNoWakeWord", "call", d.designated_no_wake_word)?, stop_words, + follow_up: get_ms(call, "followUpMs", "call", d.follow_up, 5000)?, }, chat_wake_words, )) @@ -476,7 +477,8 @@ mod tests { let s = parse_settings(&json!({ "voice": {"wakeWords": ["자비스", "javis"]}, "call": {"similarity": 0.3, "mode": "addressed", "designated": [ID], "designatedOnly": true, - "designatedNoWakeWord": true, "stopWords": ["멈춰", " "], "chatWakeWords": ["봇"]}, + "designatedNoWakeWord": true, "stopWords": ["멈춰", " "], "chatWakeWords": ["봇"], + "followUpMs": 2500}, "response": {"policy": "important", "queueMax": 3, "collectMs": 500, "settleMs": 0, "priorities": {ID: 2}} })).unwrap(); @@ -487,6 +489,7 @@ mod tests { assert!(s.call.designated_only && s.call.designated_no_wake_word); assert_eq!(s.call.stop_words, vec!["멈춰"]); assert_eq!(s.chat_wake_words, Some(vec!["봇".to_string()])); + assert_eq!(s.call.follow_up, Duration::from_millis(2500)); assert_eq!(s.response.kind, PolicyKind::Important); assert_eq!(s.response.queue_max, 3); assert_eq!(s.response.collect, Duration::from_millis(500)); @@ -499,6 +502,7 @@ mod tests { (json!({"call": {"similarity": 0.9}}), "call.similarity"), (json!({"call": {"mode": "sometimes"}}), "call.mode"), (json!({"call": {"designated": ["1"]}}), "call.designated"), + (json!({"call": {"followUpMs": 6000}}), "call.followUpMs"), (json!({"response": {"policy": "random"}}), "response.policy"), (json!({"response": {"queueMax": 0}}), "response.queueMax"), (json!({"response": {"collectMs": 5000}}), "response.collectMs"), diff --git a/docs/v2/conversation.md b/docs/v2/conversation.md index 119a21a..f3e107c 100644 --- a/docs/v2/conversation.md +++ b/docs/v2/conversation.md @@ -20,6 +20,7 @@ |---|---| | 호출어(prefix) | GUI에서 여러 개 등록·수정·삭제 (예: "자비스", "야 자비스", "javis"). 음성과 채팅에 같이 쓰고, 채팅용은 따로 지정할 수도 있습니다. 비슷한 발음 허용 정도(예: "자비즈")도 조절합니다. | | 호출 방식 | (1) 호출어로 부를 때만 응답 **(기본값)** / (2) 봇에게 하는 말로 판단되면 응답 / (3) 모든 말에 응답 | +| 호출어 뒤 쉼 | "자비스, … 지금 몇 시야?"처럼 이름 뒤에 쉬어서 "자비스"만 따로 들리면, 같은 사람이 이어서 하는 말(기본 1.5초 안에 시작)과 합쳐 한 질문으로 받습니다. 아무 말도 이어지지 않으면 "자비스"에만 대답합니다. 채팅은 합치지 않습니다. | | 지정한 사람 | GUI에서 서버 멤버 목록으로 고릅니다. 아래 두 설정을 **각각 켜고 끕니다**. | | └ 지정한 사람만 응답 | 기본: 꺼짐. 켜면 지정하지 않은 사람이 불러도 대답하지 않습니다. | | └ 지정한 사람은 호출어 없이 대답 | 기본: 꺼짐. 켜면 지정한 사람이 하는 말에는 호출어 없이도 대답합니다. 다른 사람은 "호출 방식"을 그대로 따릅니다. |