diff --git a/app/src/conversation.rs b/app/src/conversation.rs index 412360f..cc0aff5 100644 --- a/app/src/conversation.rs +++ b/app/src/conversation.rs @@ -30,6 +30,8 @@ pub trait Voice { fn stop_all(&self); /// Play the earcon now (queued like speech; dropped by a stop). fn earcon(&self); + /// A call is still being said (partial transcript): the brain may get ready to answer it. + fn prepare(&self, user: u64, text: &str); fn ask_addressed(&self, id: &str, user: u64, text: &str) -> bool; fn ask_rank(&self, id: &str, candidates: &[(String, u64, String)]) -> bool; /// Have the brain answer a chat message (not spoken); its text comes as `CoreEvent::ReplyText`. @@ -54,6 +56,9 @@ impl Voice for DiscordCore { fn earcon(&self) { DiscordCore::earcon(self); } + fn prepare(&self, user: u64, text: &str) { + DiscordCore::prepare(self, user, text); + } fn ask_addressed(&self, id: &str, user: u64, text: &str) -> bool { DiscordCore::ask_addressed(self, id, user, text) } @@ -185,6 +190,22 @@ impl Conversation { } } + /// A partial that already reads as a call (joined to a waiting wake word alone, as its final + /// would be) lets the brain get ready while the speaker is still talking. Not while Jarvis is + /// answering: the call may not be taken, and the brain is busy with the answer. + fn on_partial(&mut self, speaker: u64, partial: &str) { + if !self.answers.is_empty() { + return; + } + let text = match self.wakes.get(&speaker) { + Some((wake, _)) => format!("{} {}", wake.text, partial), + None => partial.to_owned(), + }; + if matches!(self.calls.judge(speaker, &text), Call::Request { .. }) && !self.calls.wake_word_only(&text) { + self.voice.prepare(speaker, &text); + } + } + pub fn missed(&self) -> impl Iterator { self.missed.iter() } @@ -199,6 +220,7 @@ impl Conversation { } } CoreEvent::Chat(m) => self.on_chat(m, now), + CoreEvent::Partial(h, partial) => self.on_partial(h.speaker_id, partial), CoreEvent::ReplyText { id, text, failed } => { if let Some((channel, message, asked)) = self.chat_answers.remove(id) { let late = now >= asked + CHAT_ANSWER_KEPT; @@ -447,6 +469,7 @@ mod tests { Cancel(String), StopAll, Earcon, + Prepare(u64, String), Addressed(String, u64), Rank(String, Vec), ChatAnswer(String, u64, String), @@ -474,6 +497,9 @@ mod tests { fn earcon(&self) { self.did.borrow_mut().push(Did::Earcon); } + fn prepare(&self, user: u64, text: &str) { + self.did.borrow_mut().push(Did::Prepare(user, text.into())); + } fn ask_addressed(&self, id: &str, user: u64, _text: &str) -> bool { self.did.borrow_mut().push(Did::Addressed(id.into(), user)); true @@ -553,6 +579,29 @@ mod tests { assert!(f.take().is_empty(), "missed while busy, and a wake word alone waits silently"); } + fn partial(speaker: u64, text: &str) -> CoreEvent { + CoreEvent::Partial(Heard { speaker_id: speaker, speaker_name: "x".into(), utt: 1 }, text.into()) + } + + #[test] + fn a_partial_that_reads_as_a_call_lets_the_brain_get_ready() { + let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord); + c.on_event(&partial(A, "자비스 오늘"), ms(t, 0)); + c.on_event(&partial(B, "오늘 뭐 먹지"), ms(t, 0)); // not a call + c.on_event(&partial(A, "자비스"), ms(t, 0)); // the name alone: nothing to get ready for + c.on_event(&partial(A, "자비스 멈춰"), ms(t, 0)); // a stop is not answered + assert_eq!(f.take(), vec![Did::Prepare(A, "자비스 오늘".into())]); + // After a wake word said alone, the next utterance's partial is joined to it, as its final. + c.on_event(&fin_utt(B, 1, "자비스"), ms(t, 100)); + c.on_event(&partial(B, "지금 몇 시"), ms(t, 300)); + assert_eq!(f.take(), vec![Did::Prepare(B, "자비스 지금 몇 시".into())]); + // Not while an answer runs. + c.on_event(&fin(A, "자비스 날씨", 100), ms(t, 400)); + f.take(); + c.on_event(&partial(A, "자비스 내일은"), ms(t, 500)); + assert!(f.take().is_empty()); + } + #[test] fn the_earcon_can_be_turned_off() { let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord); diff --git a/app/src/conversation.spec.md b/app/src/conversation.spec.md index f7140b9..7f0dd5d 100644 --- a/app/src/conversation.spec.md +++ b/app/src/conversation.spec.md @@ -169,9 +169,14 @@ never takes the voice turn and never goes through the response policy. `app/src/conversation.rs` runs between the core's events and everything else that reads them (every event is passed on unchanged). It talks to the core through a small `Voice` interface (answer, cancel -one request, stop all, earcon, ask "addressed", ask "rank"), so the glue is tested with a fake. +one request, stop all, earcon, prepare, ask "addressed", ask "rank"), so the glue is tested with a fake. - `Chat` and `ReplyText`: chat calls, above. +- `Partial` (getting ready): a partial transcript that call detection already reads as a request + (joined to the speaker's waiting wake word alone, as its final would be; not the wake word alone, + not a stop) is sent to the brain as `prepare(user, text)`, so the brain judges what the call needs + while the speaker is still talking (`voice_worker.spec.md`, `prepare`). Not while an answer runs: + the call may not be taken, and the brain is busy answering. - `SpeechStart` opens the speaker's utterance; `SpeakingStopped` marks it ending. - `Final`: a request gets the next id, `ended_at` = arrival minus the final's `wait_ms`, and goes through call detection. Ignore closes the utterance. Stop stops all speech, runs the policy's stop, @@ -202,7 +207,8 @@ modes) and every policy rule above with fake times (settling, ordering by end ti interruption, ranking tie, ranking timeout, stop dropping the queue), and the conversation loop with a fake voice side (answering and queueing, a busy drop recorded without text, voice stop, addressed yes, no and timeout, the earcon before each answer, once for a wake word alone, none while busy or when -off, interruption cancelling the right speech, a ranking deciding, end times from +off, partials that read as a call sent to get ready (joined to a waiting wake word, none while +answering), interruption cancelling the right speech, a ranking deciding, end times from `wait_ms`, and chat: an answer posted as a reply once, per-channel switches, no answers to bots or to a stop, chat wake words and designated people, a failed answer posting nothing). `tests/worker_protocol.rs` checks an answer spoken under its id and both judgements against the real Python worker with a fake brain. diff --git a/app/src/discord.rs b/app/src/discord.rs index 8bbe923..7a2b651 100644 --- a/app/src/discord.rs +++ b/app/src/discord.rs @@ -441,6 +441,11 @@ impl DiscordCore { self.inner.worker.addressed(id, user, text) } + /// Tell the brain a call from `user` is being said; nothing comes back. + pub fn prepare(&self, user: u64, text: &str) { + self.inner.worker.prepare(user, text); + } + /// Ask the brain to rank candidates `(id, user, text)`; the answer comes as `CoreEvent::Ranked`. pub fn ask_rank(&self, id: &str, candidates: &[(String, u64, String)]) -> bool { self.inner.worker.rank(id, candidates) diff --git a/app/src/discord_core.spec.md b/app/src/discord_core.spec.md index b145ae3..bcd5252 100644 --- a/app/src/discord_core.spec.md +++ b/app/src/discord_core.spec.md @@ -155,7 +155,8 @@ and the late "left" update cancelled the new join (seen on real Discord). from the brain, a refused `reply` (`brain_unavailable`) or a lost worker ends it as failed. `cancel(id)` stops one request (its clip if playing, the worker's unsent sentences, or the speech of an answer still being written) and the rest of the queue goes on. `ask_addressed(id, user, text)` and - `ask_rank(id, candidates)` pass the conversation's questions to the brain. + `ask_rank(id, candidates)` pass the conversation's questions to the brain. `prepare(user, text)` sends `prepare` (a + call still being said) and expects nothing back. - **Earcon** (M4): `earcon()` queues the app's own earcon clip (`app/src/earcon.rs`, 48 kHz stereo, peak 0.3 of full scale) as a request of its own: no worker round trip, `SpeakStart`/`SpeakEnd` like any request, dropped by a stop. None when not in a voice channel. diff --git a/app/src/worker_client.rs b/app/src/worker_client.rs index 79e4e19..633b4ba 100644 --- a/app/src/worker_client.rs +++ b/app/src/worker_client.rs @@ -343,6 +343,11 @@ impl WorkerClient { self.send_json(json!({"type": "addressed", "id": id, "user": user.to_string(), "text": text})) } + /// Tell the brain a call is being said (a partial transcript), so it gets ready to answer it. + pub fn prepare(&self, user: u64, text: &str) -> bool { + self.send_json(json!({"type": "prepare", "user": user.to_string(), "text": text})) + } + /// Ask the brain to rank tied candidates `(id, user, text)` by importance. pub fn rank(&self, id: &str, candidates: &[(String, u64, String)]) -> bool { let list: Vec = candidates.iter() diff --git a/app/tests/live_round_trip.rs b/app/tests/live_round_trip.rs index 9a50e34..3877064 100644 --- a/app/tests/live_round_trip.rs +++ b/app/tests/live_round_trip.rs @@ -72,7 +72,8 @@ struct Ears { jarvis: u64, ssrc: Arc>>, stream: Arc>>>, - first_audible: Arc>>, + /// When each audible 20 ms frame arrived (the earcon first, then the answer). + audible: Arc>>, } #[async_trait] @@ -95,7 +96,7 @@ impl VoiceHandler for Ears { // Audible: louder than -36 dBFS. Silence frames do not count. let rms = (f.iter().map(|&x| f64::from(x).powi(2)).sum::() / f.len().max(1) as f64).sqrt(); if rms > 500.0 { - self.first_audible.lock().unwrap().get_or_insert_with(Instant::now); + self.audible.lock().unwrap().push(Instant::now()); } } } @@ -261,7 +262,7 @@ async fn conversation_round_trip_on_real_discord() { // The previous answer is over before the next question. until("Jarvis to finish speaking", 60_000, || !core.speaking()).await; let before = events.lock().unwrap().len(); - *ears.first_audible.lock().unwrap() = None; + ears.audible.lock().unwrap().clear(); let track = call.lock().await.play_only_input(Input::from(wav.clone())); let deadline = Instant::now() + Duration::from_secs(5); let started = loop { @@ -305,8 +306,18 @@ async fn conversation_round_trip_on_real_discord() { text: finals.iter().map(|(_, f)| f.text.as_str()).collect::>().join(" | "), stt_ms: f.stt_ms, })); - let speak_start = ev.iter().find_map(|(t, e)| matches!(e, CoreEvent::SpeakStart(_)).then_some(*t)); - let first_sound = *ears.first_audible.lock().unwrap(); + // The first sound is the earcon; the answer's own sound starts after its SpeakStart. + let reply_id = ev.iter().find_map(|(_, e)| match e { + CoreEvent::ReplyText { id, .. } => Some(id.clone()), + _ => None, + }); + let speak_start = ev.iter().find_map(|(t, e)| match e { + CoreEvent::SpeakStart(id) if Some(id) == reply_id.as_ref() => Some(*t), + _ => None, + }); + let audible = ears.audible.lock().unwrap().clone(); + let first_sound = audible.first().copied(); + let answer_sound = speak_start.and_then(|s| audible.iter().copied().find(|t| *t >= s)); rows.push(json!({ "round": round, "question": text, "heard": final_ev.as_ref().map(|(_, f)| f.text.clone()), @@ -316,6 +327,7 @@ async fn conversation_round_trip_on_real_discord() { "answer_text_ms": ms(reply.as_ref().map(|r| r.0)), "speak_start_ms": ms(speak_start), "first_sound_ms": ms(first_sound), + "answer_sound_ms": ms(answer_sound), "spoken": finished, })); println!("{phase} {}", rows.last().unwrap()); @@ -324,12 +336,17 @@ async fn conversation_round_trip_on_real_discord() { } stop.store(true, Ordering::Relaxed); let load_finals = match load { Some(h) => Some(h.await.unwrap()), None => None }; - let mut sound: Vec = rows.iter().filter_map(|r| r["first_sound_ms"].as_i64()).collect(); - sound.sort_unstable(); - let pick = |q: f64| sound.get(((sound.len() as f64 - 1.0) * q).round() as usize).copied(); + let stats = |key: &str| { + let mut v: Vec = rows.iter().filter_map(|r| r[key].as_i64()).collect(); + v.sort_unstable(); + let pick = |q: f64| v.get(((v.len() as f64 - 1.0) * q).round() as usize).copied(); + (v.len(), json!({"min": v.first(), "median": pick(0.5), "p90": pick(0.9), "max": v.last()})) + }; + let (answered, answer_sound) = stats("answer_sound_ms"); report.insert(phase.into(), json!({ - "rows": rows, "answered": sound.len(), "asked": rows.len(), - "first_sound_ms": {"min": sound.first(), "median": pick(0.5), "p90": pick(0.9), "max": sound.last()}, + "rows": rows, "answered": answered, "asked": rows.len(), + "first_sound_ms": stats("first_sound_ms").1, + "answer_sound_ms": answer_sound, "background_finals": load_finals, })); }