fix(app): a wake word said alone waits for what the speaker says next
A pause after the name ("자비스, ... 지금 몇 시야?") is longer than the
worker's 300 ms end silence, so the wake word ended its own utterance:
Jarvis answered the bare name and ignored the question that followed (no
wake word). Found on real Discord. A voice utterance that is only a wake
word is now joined to the same speaker's next final when that utterance
is running or starts within call.followUpMs (default 1500); otherwise the
wake word alone is the request after that time. Stops drop it.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -4,6 +4,7 @@
|
||||
//! matching works on Unicode-normalised letters, so it behaves the same in any script.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::time::Duration;
|
||||
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
|
||||
@@ -27,6 +28,9 @@ pub struct CallSettings {
|
||||
pub designated_only: bool,
|
||||
pub designated_no_wake_word: bool,
|
||||
pub stop_words: Vec<String>,
|
||||
/// After a wake word said alone, how long the speaker's next utterance may take to start and
|
||||
/// still be joined to it (the pause after "Jarvis," often ends an utterance).
|
||||
pub follow_up: Duration,
|
||||
}
|
||||
|
||||
impl Default for CallSettings {
|
||||
@@ -39,6 +43,7 @@ impl Default for CallSettings {
|
||||
designated_only: false,
|
||||
designated_no_wake_word: false,
|
||||
stop_words: ["멈춰", "그만", "정지", "stop"].map(String::from).to_vec(),
|
||||
follow_up: Duration::from_millis(1500),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -122,6 +127,11 @@ impl CallSettings {
|
||||
})
|
||||
}
|
||||
|
||||
/// A wake word is heard and nothing else (no letters or digits around it).
|
||||
pub fn wake_word_only(&self, text: &str) -> bool {
|
||||
self.without_wake_word(&normalise(text)).is_some_and(|rest| rest.is_empty())
|
||||
}
|
||||
|
||||
/// Decide what to do with one finished utterance.
|
||||
pub fn judge(&self, speaker: u64, text: &str) -> Call {
|
||||
let norm = normalise(text);
|
||||
@@ -160,6 +170,16 @@ mod tests {
|
||||
CallSettings::default()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_wake_word_with_nothing_else_is_told_apart() {
|
||||
let c = s();
|
||||
assert!(c.wake_word_only("자비스,"));
|
||||
assert!(c.wake_word_only("자비즈?"));
|
||||
assert!(!c.wake_word_only("자비스, 지금 몇 시야?"));
|
||||
assert!(!c.wake_word_only("서비스"));
|
||||
assert!(!c.wake_word_only(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_and_near_wake_words_call_but_a_different_word_does_not() {
|
||||
let c = s();
|
||||
|
||||
@@ -130,13 +130,18 @@ pub struct Conversation<V: Voice> {
|
||||
chat: ChatCalls,
|
||||
/// Chat answer id -> the message it answers (channel, message) and when it was asked.
|
||||
chat_answers: HashMap<String, (u64, u64, Instant)>,
|
||||
/// Each speaker's latest utterance that started.
|
||||
started: HashMap<u64, u32>,
|
||||
/// A wake word said alone, per speaker, waiting for what they say next; the deadline for the next
|
||||
/// utterance to start (None while it runs).
|
||||
wakes: HashMap<u64, (Request, Option<Instant>)>,
|
||||
}
|
||||
|
||||
impl<V: Voice> Conversation<V> {
|
||||
pub fn new(voice: V, calls: CallSettings, policy: PolicySettings) -> Self {
|
||||
Self { voice, calls, policy: Policy::new(policy), next_id: 0, answers: HashMap::new(),
|
||||
by_speech: HashMap::new(), asks: HashMap::new(), missed: VecDeque::new(), chat: ChatCalls::default(),
|
||||
chat_answers: HashMap::new() }
|
||||
chat_answers: HashMap::new(), started: HashMap::new(), wakes: HashMap::new() }
|
||||
}
|
||||
|
||||
/// Answer the registered chat channels too.
|
||||
@@ -178,7 +183,13 @@ impl<V: Voice> Conversation<V> {
|
||||
|
||||
pub fn on_event(&mut self, ev: &CoreEvent, now: Instant) {
|
||||
match ev {
|
||||
CoreEvent::SpeechStart(h) => self.policy.utterance_open(h.speaker_id),
|
||||
CoreEvent::SpeechStart(h) => {
|
||||
self.policy.utterance_open(h.speaker_id);
|
||||
self.started.insert(h.speaker_id, h.utt);
|
||||
if let Some((_, deadline)) = self.wakes.get_mut(&h.speaker_id) {
|
||||
*deadline = None; // they go on: wait for this utterance's final
|
||||
}
|
||||
}
|
||||
CoreEvent::Chat(m) => self.on_chat(m, now),
|
||||
CoreEvent::ReplyText { id, text, failed } => {
|
||||
if let Some((channel, message, asked)) = self.chat_answers.remove(id) {
|
||||
@@ -195,11 +206,27 @@ impl<V: Voice> Conversation<V> {
|
||||
CoreEvent::SpeakingStopped(user) => self.policy.speaking_stopped(*user, now),
|
||||
CoreEvent::Final(f) => {
|
||||
let speaker = f.heard.speaker_id;
|
||||
// What follows a wake word said alone is joined to it.
|
||||
let text = match self.wakes.remove(&speaker) {
|
||||
Some((wake, _)) => format!("{} {}", wake.text, f.text),
|
||||
None => f.text.clone(),
|
||||
};
|
||||
self.next_id += 1;
|
||||
let req = Request { id: self.next_id, speaker, text: f.text.clone(), direct: false,
|
||||
let req = Request { id: self.next_id, speaker, text: text.clone(), direct: false,
|
||||
ended_at: now.checked_sub(Duration::from_millis(f.wait_ms)).unwrap_or(now), arrived_at: now };
|
||||
let call = self.calls.judge(speaker, &f.text);
|
||||
let call = self.calls.judge(speaker, &text);
|
||||
debug_log!("conversation", "final speaker={speaker} req={} call={call:?}", req.id);
|
||||
if matches!(call, Call::Request { direct: true }) && self.calls.wake_word_only(&text) {
|
||||
let going_on = self.started.get(&speaker).is_some_and(|&u| u > f.heard.utt);
|
||||
debug_log!("conversation", "wake word alone req={} speaker={speaker} going_on={going_on}", req.id);
|
||||
if !going_on {
|
||||
let a = self.policy.utterance_closed(speaker, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
let deadline = (!going_on).then(|| now + self.calls.follow_up);
|
||||
self.wakes.insert(speaker, (Request { direct: true, ..req }, deadline));
|
||||
return;
|
||||
}
|
||||
match call {
|
||||
Call::Ignore => {
|
||||
let a = self.policy.utterance_closed(speaker, now);
|
||||
@@ -217,7 +244,7 @@ impl<V: Voice> Conversation<V> {
|
||||
}
|
||||
Call::Ask => {
|
||||
let id = format!("q{}", req.id);
|
||||
if self.voice.ask_addressed(&id, speaker, &f.text) {
|
||||
if self.voice.ask_addressed(&id, speaker, &text) {
|
||||
self.asks.insert(id, (req, now));
|
||||
} else {
|
||||
let a = self.policy.utterance_closed(speaker, now);
|
||||
@@ -263,6 +290,7 @@ impl<V: Voice> Conversation<V> {
|
||||
|
||||
/// The policy's stop, and every pending "was it said to Jarvis?" question is void.
|
||||
fn stop_everything(&mut self, now: Instant) {
|
||||
self.wakes.clear();
|
||||
let a = self.policy.stop();
|
||||
self.apply(a, now);
|
||||
for (_, (req, _)) in self.asks.drain().collect::<Vec<_>>() {
|
||||
@@ -282,16 +310,24 @@ impl<V: Voice> Conversation<V> {
|
||||
self.apply(a, now);
|
||||
}
|
||||
}
|
||||
// Nothing followed a wake word said alone: it is the request.
|
||||
let due: Vec<u64> = self.wakes.iter()
|
||||
.filter(|(_, (_, d))| d.is_some_and(|d| now >= d)).map(|(s, _)| *s).collect();
|
||||
for speaker in due {
|
||||
if let Some((req, _)) = self.wakes.remove(&speaker) {
|
||||
debug_log!("conversation", "wake word alone req={} answered alone", req.id);
|
||||
let a = self.policy.request(req, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
}
|
||||
let a = self.policy.tick(now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
|
||||
pub fn next_deadline(&self) -> Option<Instant> {
|
||||
let asks = self.asks.values().map(|(_, at)| *at + ADDRESSED_TIMEOUT).min();
|
||||
match (self.policy.next_deadline(), asks) {
|
||||
(Some(a), Some(b)) => Some(a.min(b)),
|
||||
(a, b) => a.or(b),
|
||||
}
|
||||
let asks = self.asks.values().map(|(_, at)| *at + ADDRESSED_TIMEOUT);
|
||||
let wakes = self.wakes.values().filter_map(|(_, d)| *d);
|
||||
self.policy.next_deadline().into_iter().chain(asks).chain(wakes).min()
|
||||
}
|
||||
|
||||
fn apply(&mut self, actions: Vec<Action>, now: Instant) {
|
||||
@@ -440,6 +476,71 @@ mod tests {
|
||||
t + Duration::from_millis(n)
|
||||
}
|
||||
|
||||
fn fin_utt(speaker: u64, utt: u32, text: &str) -> CoreEvent {
|
||||
CoreEvent::Final(HeardFinal { heard: Heard { speaker_id: speaker, speaker_name: "x".into(), utt },
|
||||
text: text.into(), end_reason: "silence".into(), audio_ms: 1000, wait_ms: 100, stt_ms: 100 })
|
||||
}
|
||||
fn start(speaker: u64, utt: u32) -> CoreEvent {
|
||||
CoreEvent::SpeechStart(Heard { speaker_id: speaker, speaker_name: "x".into(), utt })
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_wake_word_alone_waits_for_what_the_speaker_says_next() {
|
||||
let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord);
|
||||
// "자비스," then a pause that ended the utterance; the question follows.
|
||||
c.on_event(&start(A, 1), ms(t, 0));
|
||||
c.on_event(&fin_utt(A, 1, "자비스,"), ms(t, 900));
|
||||
assert!(f.take().is_empty(), "a wake word alone is not the question yet");
|
||||
c.on_event(&start(A, 2), ms(t, 1200));
|
||||
c.tick(ms(t, 5000)); // still talking: no time limit while the next utterance runs
|
||||
assert!(f.take().is_empty());
|
||||
c.on_event(&fin_utt(A, 2, "지금 몇 시야?"), ms(t, 5200));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스, 지금 몇 시야?".into())]);
|
||||
assert_eq!(c.missed().count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_wake_word_alone_waits_for_a_next_utterance_that_already_started() {
|
||||
let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord);
|
||||
c.on_event(&start(A, 1), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 600)); // the final of utterance 1 comes after 2 started
|
||||
c.on_event(&fin_utt(A, 1, "자비스"), ms(t, 900));
|
||||
c.tick(ms(t, 900) + CallSettings::default().follow_up + Duration::from_millis(1));
|
||||
assert!(f.take().is_empty());
|
||||
c.on_event(&fin_utt(A, 2, "멈춰"), ms(t, 3000));
|
||||
assert_eq!(f.take(), vec![Did::StopAll], "joined, it is a stop");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_wake_word_alone_is_answered_when_nothing_follows() {
|
||||
let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord);
|
||||
c.on_event(&start(A, 1), ms(t, 0));
|
||||
c.on_event(&fin_utt(A, 1, "자비스"), ms(t, 900));
|
||||
let wait = CallSettings::default().follow_up;
|
||||
assert_eq!(c.next_deadline(), Some(ms(t, 900) + wait));
|
||||
c.tick(ms(t, 900) + wait - Duration::from_millis(1));
|
||||
assert!(f.take().is_empty());
|
||||
c.tick(ms(t, 900) + wait);
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스".into())]);
|
||||
// Another speaker's words do not join it.
|
||||
c.on_event(&end("s1", false), ms(t, 4000));
|
||||
c.on_event(&start(A, 2), ms(t, 4100));
|
||||
c.on_event(&fin_utt(A, 2, "자비스"), ms(t, 4500));
|
||||
c.on_event(&fin_utt(B, 1, "오늘 뭐 먹지"), ms(t, 4600));
|
||||
c.tick(ms(t, 4500) + wait);
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stop_drops_a_waiting_wake_word() {
|
||||
let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord);
|
||||
c.on_event(&fin_utt(A, 1, "자비스"), ms(t, 900));
|
||||
c.on_event(&CoreEvent::Stopped, ms(t, 1000));
|
||||
c.tick(ms(t, 900) + CallSettings::default().follow_up);
|
||||
assert!(f.take().is_empty());
|
||||
assert_eq!(c.next_deadline(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_call_is_answered_and_the_next_one_waits_for_its_speech_to_end() {
|
||||
let (mut c, f, t) = conv(PolicyKind::InOrder, CallMode::WakeWord);
|
||||
|
||||
@@ -32,6 +32,7 @@ Settings (`settings.json`, section `call`; wake words stay in `voice.wakeWords`)
|
||||
| `call.designatedOnly` | `false` | Answer only designated people. |
|
||||
| `call.designatedNoWakeWord` | `false` | Answer designated people without a wake word. |
|
||||
| `call.stopWords` | `["멈춰", "그만", "정지", "stop"]` | Said together with a wake word, they stop Jarvis in any policy. |
|
||||
| `call.followUpMs` | `1500` | After a wake word said alone, how long the speaker's next utterance may take to start and still be joined to it. 0-5000. |
|
||||
|
||||
Normalisation: Unicode NFKD, lower case, and only letters and digits kept (spaces, punctuation and
|
||||
combining marks dropped). NFKD splits Hangul syllables into their letters, so "자비즈" is one letter
|
||||
@@ -54,6 +55,17 @@ Decision for one utterance (`judge(speaker, text)`), in order:
|
||||
|
||||
The text sent to the brain is the utterance as heard; the brain treats the wake word as address.
|
||||
|
||||
**A wake word said alone.** People often pause after the name ("자비스, ... 지금 몇 시야?"), and a
|
||||
pause longer than the worker's `end_silence_ms` ends the utterance there. A voice utterance that is a
|
||||
wake word and nothing else (no letters or digits left once the wake word is removed) is therefore
|
||||
not a request yet: the conversation keeps it for that speaker and joins it to their next final, with
|
||||
a space between, and judges the joined text as one utterance (so "자비스" then "멈춰" is a stop). If
|
||||
the speaker's next utterance already started when the wake word's final came, or starts within
|
||||
`call.followUpMs` of it, the wake word waits for that utterance's final however long it takes;
|
||||
otherwise, after `call.followUpMs`, the wake word alone is the request. Other speakers' utterances
|
||||
never join it, and a stop (voice, slash command or leaving) drops it. Chat messages are not joined:
|
||||
a message is complete as sent.
|
||||
|
||||
## Response policy
|
||||
|
||||
Settings (section `response`):
|
||||
|
||||
@@ -242,6 +242,7 @@ fn parse_call(raw: &Map<String, Value>, wake_words: &[String], d: &CallSettings)
|
||||
designated_only: get_bool(call, "designatedOnly", "call", d.designated_only)?,
|
||||
designated_no_wake_word: get_bool(call, "designatedNoWakeWord", "call", d.designated_no_wake_word)?,
|
||||
stop_words,
|
||||
follow_up: get_ms(call, "followUpMs", "call", d.follow_up, 5000)?,
|
||||
},
|
||||
chat_wake_words,
|
||||
))
|
||||
@@ -476,7 +477,8 @@ mod tests {
|
||||
let s = parse_settings(&json!({
|
||||
"voice": {"wakeWords": ["자비스", "javis"]},
|
||||
"call": {"similarity": 0.3, "mode": "addressed", "designated": [ID], "designatedOnly": true,
|
||||
"designatedNoWakeWord": true, "stopWords": ["멈춰", " "], "chatWakeWords": ["봇"]},
|
||||
"designatedNoWakeWord": true, "stopWords": ["멈춰", " "], "chatWakeWords": ["봇"],
|
||||
"followUpMs": 2500},
|
||||
"response": {"policy": "important", "queueMax": 3, "collectMs": 500, "settleMs": 0,
|
||||
"priorities": {ID: 2}}
|
||||
})).unwrap();
|
||||
@@ -487,6 +489,7 @@ mod tests {
|
||||
assert!(s.call.designated_only && s.call.designated_no_wake_word);
|
||||
assert_eq!(s.call.stop_words, vec!["멈춰"]);
|
||||
assert_eq!(s.chat_wake_words, Some(vec!["봇".to_string()]));
|
||||
assert_eq!(s.call.follow_up, Duration::from_millis(2500));
|
||||
assert_eq!(s.response.kind, PolicyKind::Important);
|
||||
assert_eq!(s.response.queue_max, 3);
|
||||
assert_eq!(s.response.collect, Duration::from_millis(500));
|
||||
@@ -499,6 +502,7 @@ mod tests {
|
||||
(json!({"call": {"similarity": 0.9}}), "call.similarity"),
|
||||
(json!({"call": {"mode": "sometimes"}}), "call.mode"),
|
||||
(json!({"call": {"designated": ["1"]}}), "call.designated"),
|
||||
(json!({"call": {"followUpMs": 6000}}), "call.followUpMs"),
|
||||
(json!({"response": {"policy": "random"}}), "response.policy"),
|
||||
(json!({"response": {"queueMax": 0}}), "response.queueMax"),
|
||||
(json!({"response": {"collectMs": 5000}}), "response.collectMs"),
|
||||
|
||||
Reference in New Issue
Block a user