fix(app): a wake word said alone waits for what the speaker says next

A pause after the name ("자비스, ... 지금 몇 시야?") is longer than the
worker's 300 ms end silence, so the wake word ended its own utterance:
Jarvis answered the bare name and ignored the question that followed (no
wake word). Found on real Discord. A voice utterance that is only a wake
word is now joined to the same speaker's next final when that utterance
is running or starts within call.followUpMs (default 1500); otherwise the
wake word alone is the request after that time. Stops drop it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-12 09:44:06 +09:00
parent 9ff307d7fe
commit bd2eb49e3d
5 changed files with 149 additions and 11 deletions

View File

@@ -4,6 +4,7 @@
//! matching works on Unicode-normalised letters, so it behaves the same in any script.
use std::collections::HashSet;
use std::time::Duration;
use unicode_normalization::UnicodeNormalization;
@@ -27,6 +28,9 @@ pub struct CallSettings {
pub designated_only: bool,
pub designated_no_wake_word: bool,
pub stop_words: Vec<String>,
/// After a wake word said alone, how long the speaker's next utterance may take to start and
/// still be joined to it (the pause after "Jarvis," often ends an utterance).
pub follow_up: Duration,
}
impl Default for CallSettings {
@@ -39,6 +43,7 @@ impl Default for CallSettings {
designated_only: false,
designated_no_wake_word: false,
stop_words: ["멈춰", "그만", "정지", "stop"].map(String::from).to_vec(),
follow_up: Duration::from_millis(1500),
}
}
}
@@ -122,6 +127,11 @@ impl CallSettings {
})
}
/// A wake word is heard and nothing else (no letters or digits around it).
pub fn wake_word_only(&self, text: &str) -> bool {
self.without_wake_word(&normalise(text)).is_some_and(|rest| rest.is_empty())
}
/// Decide what to do with one finished utterance.
pub fn judge(&self, speaker: u64, text: &str) -> Call {
let norm = normalise(text);
@@ -160,6 +170,16 @@ mod tests {
CallSettings::default()
}
#[test]
fn a_wake_word_with_nothing_else_is_told_apart() {
let c = s();
assert!(c.wake_word_only("자비스,"));
assert!(c.wake_word_only("자비즈?"));
assert!(!c.wake_word_only("자비스, 지금 몇 시야?"));
assert!(!c.wake_word_only("서비스"));
assert!(!c.wake_word_only(""));
}
#[test]
fn exact_and_near_wake_words_call_but_a_different_word_does_not() {
let c = s();

View File

@@ -130,13 +130,18 @@ pub struct Conversation<V: Voice> {
chat: ChatCalls,
/// Chat answer id -> the message it answers (channel, message) and when it was asked.
chat_answers: HashMap<String, (u64, u64, Instant)>,
/// Each speaker's latest utterance that started.
started: HashMap<u64, u32>,
/// A wake word said alone, per speaker, waiting for what they say next; the deadline for the next
/// utterance to start (None while it runs).
wakes: HashMap<u64, (Request, Option<Instant>)>,
}
impl<V: Voice> Conversation<V> {
pub fn new(voice: V, calls: CallSettings, policy: PolicySettings) -> Self {
Self { voice, calls, policy: Policy::new(policy), next_id: 0, answers: HashMap::new(),
by_speech: HashMap::new(), asks: HashMap::new(), missed: VecDeque::new(), chat: ChatCalls::default(),
chat_answers: HashMap::new() }
chat_answers: HashMap::new(), started: HashMap::new(), wakes: HashMap::new() }
}
/// Answer the registered chat channels too.
@@ -178,7 +183,13 @@ impl<V: Voice> Conversation<V> {
pub fn on_event(&mut self, ev: &CoreEvent, now: Instant) {
match ev {
CoreEvent::SpeechStart(h) => self.policy.utterance_open(h.speaker_id),
CoreEvent::SpeechStart(h) => {
self.policy.utterance_open(h.speaker_id);
self.started.insert(h.speaker_id, h.utt);
if let Some((_, deadline)) = self.wakes.get_mut(&h.speaker_id) {
*deadline = None; // they go on: wait for this utterance's final
}
}
CoreEvent::Chat(m) => self.on_chat(m, now),
CoreEvent::ReplyText { id, text, failed } => {
if let Some((channel, message, asked)) = self.chat_answers.remove(id) {
@@ -195,11 +206,27 @@ impl<V: Voice> Conversation<V> {
CoreEvent::SpeakingStopped(user) => self.policy.speaking_stopped(*user, now),
CoreEvent::Final(f) => {
let speaker = f.heard.speaker_id;
// What follows a wake word said alone is joined to it.
let text = match self.wakes.remove(&speaker) {
Some((wake, _)) => format!("{} {}", wake.text, f.text),
None => f.text.clone(),
};
self.next_id += 1;
let req = Request { id: self.next_id, speaker, text: f.text.clone(), direct: false,
let req = Request { id: self.next_id, speaker, text: text.clone(), direct: false,
ended_at: now.checked_sub(Duration::from_millis(f.wait_ms)).unwrap_or(now), arrived_at: now };
let call = self.calls.judge(speaker, &f.text);
let call = self.calls.judge(speaker, &text);
debug_log!("conversation", "final speaker={speaker} req={} call={call:?}", req.id);
if matches!(call, Call::Request { direct: true }) && self.calls.wake_word_only(&text) {
let going_on = self.started.get(&speaker).is_some_and(|&u| u > f.heard.utt);
debug_log!("conversation", "wake word alone req={} speaker={speaker} going_on={going_on}", req.id);
if !going_on {
let a = self.policy.utterance_closed(speaker, now);
self.apply(a, now);
}
let deadline = (!going_on).then(|| now + self.calls.follow_up);
self.wakes.insert(speaker, (Request { direct: true, ..req }, deadline));
return;
}
match call {
Call::Ignore => {
let a = self.policy.utterance_closed(speaker, now);
@@ -217,7 +244,7 @@ impl<V: Voice> Conversation<V> {
}
Call::Ask => {
let id = format!("q{}", req.id);
if self.voice.ask_addressed(&id, speaker, &f.text) {
if self.voice.ask_addressed(&id, speaker, &text) {
self.asks.insert(id, (req, now));
} else {
let a = self.policy.utterance_closed(speaker, now);
@@ -263,6 +290,7 @@ impl<V: Voice> Conversation<V> {
/// The policy's stop, and every pending "was it said to Jarvis?" question is void.
fn stop_everything(&mut self, now: Instant) {
self.wakes.clear();
let a = self.policy.stop();
self.apply(a, now);
for (_, (req, _)) in self.asks.drain().collect::<Vec<_>>() {
@@ -282,16 +310,24 @@ impl<V: Voice> Conversation<V> {
self.apply(a, now);
}
}
// Nothing followed a wake word said alone: it is the request.
let due: Vec<u64> = self.wakes.iter()
.filter(|(_, (_, d))| d.is_some_and(|d| now >= d)).map(|(s, _)| *s).collect();
for speaker in due {
if let Some((req, _)) = self.wakes.remove(&speaker) {
debug_log!("conversation", "wake word alone req={} answered alone", req.id);
let a = self.policy.request(req, now);
self.apply(a, now);
}
}
let a = self.policy.tick(now);
self.apply(a, now);
}
pub fn next_deadline(&self) -> Option<Instant> {
let asks = self.asks.values().map(|(_, at)| *at + ADDRESSED_TIMEOUT).min();
match (self.policy.next_deadline(), asks) {
(Some(a), Some(b)) => Some(a.min(b)),
(a, b) => a.or(b),
}
let asks = self.asks.values().map(|(_, at)| *at + ADDRESSED_TIMEOUT);
let wakes = self.wakes.values().filter_map(|(_, d)| *d);
self.policy.next_deadline().into_iter().chain(asks).chain(wakes).min()
}
fn apply(&mut self, actions: Vec<Action>, now: Instant) {
@@ -440,6 +476,71 @@ mod tests {
t + Duration::from_millis(n)
}
fn fin_utt(speaker: u64, utt: u32, text: &str) -> CoreEvent {
CoreEvent::Final(HeardFinal { heard: Heard { speaker_id: speaker, speaker_name: "x".into(), utt },
text: text.into(), end_reason: "silence".into(), audio_ms: 1000, wait_ms: 100, stt_ms: 100 })
}
fn start(speaker: u64, utt: u32) -> CoreEvent {
CoreEvent::SpeechStart(Heard { speaker_id: speaker, speaker_name: "x".into(), utt })
}
#[test]
fn a_wake_word_alone_waits_for_what_the_speaker_says_next() {
let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord);
// "자비스," then a pause that ended the utterance; the question follows.
c.on_event(&start(A, 1), ms(t, 0));
c.on_event(&fin_utt(A, 1, "자비스,"), ms(t, 900));
assert!(f.take().is_empty(), "a wake word alone is not the question yet");
c.on_event(&start(A, 2), ms(t, 1200));
c.tick(ms(t, 5000)); // still talking: no time limit while the next utterance runs
assert!(f.take().is_empty());
c.on_event(&fin_utt(A, 2, "지금 몇 시야?"), ms(t, 5200));
assert_eq!(f.take(), vec![Did::Answer(A, "자비스, 지금 몇 시야?".into())]);
assert_eq!(c.missed().count(), 0);
}
#[test]
fn a_wake_word_alone_waits_for_a_next_utterance_that_already_started() {
let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord);
c.on_event(&start(A, 1), ms(t, 0));
c.on_event(&start(A, 2), ms(t, 600)); // the final of utterance 1 comes after 2 started
c.on_event(&fin_utt(A, 1, "자비스"), ms(t, 900));
c.tick(ms(t, 900) + CallSettings::default().follow_up + Duration::from_millis(1));
assert!(f.take().is_empty());
c.on_event(&fin_utt(A, 2, "멈춰"), ms(t, 3000));
assert_eq!(f.take(), vec![Did::StopAll], "joined, it is a stop");
}
#[test]
fn a_wake_word_alone_is_answered_when_nothing_follows() {
let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord);
c.on_event(&start(A, 1), ms(t, 0));
c.on_event(&fin_utt(A, 1, "자비스"), ms(t, 900));
let wait = CallSettings::default().follow_up;
assert_eq!(c.next_deadline(), Some(ms(t, 900) + wait));
c.tick(ms(t, 900) + wait - Duration::from_millis(1));
assert!(f.take().is_empty());
c.tick(ms(t, 900) + wait);
assert_eq!(f.take(), vec![Did::Answer(A, "자비스".into())]);
// Another speaker's words do not join it.
c.on_event(&end("s1", false), ms(t, 4000));
c.on_event(&start(A, 2), ms(t, 4100));
c.on_event(&fin_utt(A, 2, "자비스"), ms(t, 4500));
c.on_event(&fin_utt(B, 1, "오늘 뭐 먹지"), ms(t, 4600));
c.tick(ms(t, 4500) + wait);
assert_eq!(f.take(), vec![Did::Answer(A, "자비스".into())]);
}
#[test]
fn a_stop_drops_a_waiting_wake_word() {
let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord);
c.on_event(&fin_utt(A, 1, "자비스"), ms(t, 900));
c.on_event(&CoreEvent::Stopped, ms(t, 1000));
c.tick(ms(t, 900) + CallSettings::default().follow_up);
assert!(f.take().is_empty());
assert_eq!(c.next_deadline(), None);
}
#[test]
fn a_call_is_answered_and_the_next_one_waits_for_its_speech_to_end() {
let (mut c, f, t) = conv(PolicyKind::InOrder, CallMode::WakeWord);

View File

@@ -32,6 +32,7 @@ Settings (`settings.json`, section `call`; wake words stay in `voice.wakeWords`)
| `call.designatedOnly` | `false` | Answer only designated people. |
| `call.designatedNoWakeWord` | `false` | Answer designated people without a wake word. |
| `call.stopWords` | `["멈춰", "그만", "정지", "stop"]` | Said together with a wake word, they stop Jarvis in any policy. |
| `call.followUpMs` | `1500` | After a wake word said alone, how long the speaker's next utterance may take to start and still be joined to it. 0-5000. |
Normalisation: Unicode NFKD, lower case, and only letters and digits kept (spaces, punctuation and
combining marks dropped). NFKD splits Hangul syllables into their letters, so "자비즈" is one letter
@@ -54,6 +55,17 @@ Decision for one utterance (`judge(speaker, text)`), in order:
The text sent to the brain is the utterance as heard; the brain treats the wake word as address.
**A wake word said alone.** People often pause after the name ("자비스, ... 지금 몇 시야?"), and a
pause longer than the worker's `end_silence_ms` ends the utterance there. A voice utterance that is a
wake word and nothing else (no letters or digits left once the wake word is removed) is therefore
not a request yet: the conversation keeps it for that speaker and joins it to their next final, with
a space between, and judges the joined text as one utterance (so "자비스" then "멈춰" is a stop). If
the speaker's next utterance already started when the wake word's final came, or starts within
`call.followUpMs` of it, the wake word waits for that utterance's final however long it takes;
otherwise, after `call.followUpMs`, the wake word alone is the request. Other speakers' utterances
never join it, and a stop (voice, slash command or leaving) drops it. Chat messages are not joined:
a message is complete as sent.
## Response policy
Settings (section `response`):

View File

@@ -242,6 +242,7 @@ fn parse_call(raw: &Map<String, Value>, wake_words: &[String], d: &CallSettings)
designated_only: get_bool(call, "designatedOnly", "call", d.designated_only)?,
designated_no_wake_word: get_bool(call, "designatedNoWakeWord", "call", d.designated_no_wake_word)?,
stop_words,
follow_up: get_ms(call, "followUpMs", "call", d.follow_up, 5000)?,
},
chat_wake_words,
))
@@ -476,7 +477,8 @@ mod tests {
let s = parse_settings(&json!({
"voice": {"wakeWords": ["자비스", "javis"]},
"call": {"similarity": 0.3, "mode": "addressed", "designated": [ID], "designatedOnly": true,
"designatedNoWakeWord": true, "stopWords": ["멈춰", " "], "chatWakeWords": ["봇"]},
"designatedNoWakeWord": true, "stopWords": ["멈춰", " "], "chatWakeWords": ["봇"],
"followUpMs": 2500},
"response": {"policy": "important", "queueMax": 3, "collectMs": 500, "settleMs": 0,
"priorities": {ID: 2}}
})).unwrap();
@@ -487,6 +489,7 @@ mod tests {
assert!(s.call.designated_only && s.call.designated_no_wake_word);
assert_eq!(s.call.stop_words, vec!["멈춰"]);
assert_eq!(s.chat_wake_words, Some(vec!["봇".to_string()]));
assert_eq!(s.call.follow_up, Duration::from_millis(2500));
assert_eq!(s.response.kind, PolicyKind::Important);
assert_eq!(s.response.queue_max, 3);
assert_eq!(s.response.collect, Duration::from_millis(500));
@@ -499,6 +502,7 @@ mod tests {
(json!({"call": {"similarity": 0.9}}), "call.similarity"),
(json!({"call": {"mode": "sometimes"}}), "call.mode"),
(json!({"call": {"designated": ["1"]}}), "call.designated"),
(json!({"call": {"followUpMs": 6000}}), "call.followUpMs"),
(json!({"response": {"policy": "random"}}), "response.policy"),
(json!({"response": {"queueMax": 0}}), "response.queueMax"),
(json!({"response": {"collectMs": 5000}}), "response.collectMs"),