From b88e20134f62fee244163a9628e94970ffb196e6 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Thu, 12 Nov 2026 12:25:15 +0900 Subject: [PATCH] feat(app): a call still being said lets the brain get ready A partial transcript that call detection already reads as a request is sent to the worker as prepare, so the intent judge runs while the speaker talks. The live round trip now times the earcon and the answer's own sound apart. Co-Authored-By: Claude Opus 5.5 --- app/src/conversation.rs | 49 ++++++++++++++++++++++++++++++++++++ app/src/conversation.spec.md | 10 ++++++-- app/src/discord.rs | 5 ++++ app/src/discord_core.spec.md | 3 ++- app/src/worker_client.rs | 5 ++++ app/tests/live_round_trip.rs | 37 +++++++++++++++++++-------- 6 files changed, 96 insertions(+), 13 deletions(-) diff --git a/app/src/conversation.rs b/app/src/conversation.rs index 412360f..cc0aff5 100644 --- a/app/src/conversation.rs +++ b/app/src/conversation.rs @@ -30,6 +30,8 @@ pub trait Voice { fn stop_all(&self); /// Play the earcon now (queued like speech; dropped by a stop). fn earcon(&self); + /// A call is still being said (partial transcript): the brain may get ready to answer it. + fn prepare(&self, user: u64, text: &str); fn ask_addressed(&self, id: &str, user: u64, text: &str) -> bool; fn ask_rank(&self, id: &str, candidates: &[(String, u64, String)]) -> bool; /// Have the brain answer a chat message (not spoken); its text comes as `CoreEvent::ReplyText`. @@ -54,6 +56,9 @@ impl Voice for DiscordCore { fn earcon(&self) { DiscordCore::earcon(self); } + fn prepare(&self, user: u64, text: &str) { + DiscordCore::prepare(self, user, text); + } fn ask_addressed(&self, id: &str, user: u64, text: &str) -> bool { DiscordCore::ask_addressed(self, id, user, text) } @@ -185,6 +190,22 @@ impl Conversation { } } + /// A partial that already reads as a call (joined to a waiting wake word alone, as its final + /// would be) lets the brain get ready while the speaker is still talking. Not while Jarvis is + /// answering: the call may not be taken, and the brain is busy with the answer. + fn on_partial(&mut self, speaker: u64, partial: &str) { + if !self.answers.is_empty() { + return; + } + let text = match self.wakes.get(&speaker) { + Some((wake, _)) => format!("{} {}", wake.text, partial), + None => partial.to_owned(), + }; + if matches!(self.calls.judge(speaker, &text), Call::Request { .. }) && !self.calls.wake_word_only(&text) { + self.voice.prepare(speaker, &text); + } + } + pub fn missed(&self) -> impl Iterator { self.missed.iter() } @@ -199,6 +220,7 @@ impl Conversation { } } CoreEvent::Chat(m) => self.on_chat(m, now), + CoreEvent::Partial(h, partial) => self.on_partial(h.speaker_id, partial), CoreEvent::ReplyText { id, text, failed } => { if let Some((channel, message, asked)) = self.chat_answers.remove(id) { let late = now >= asked + CHAT_ANSWER_KEPT; @@ -447,6 +469,7 @@ mod tests { Cancel(String), StopAll, Earcon, + Prepare(u64, String), Addressed(String, u64), Rank(String, Vec), ChatAnswer(String, u64, String), @@ -474,6 +497,9 @@ mod tests { fn earcon(&self) { self.did.borrow_mut().push(Did::Earcon); } + fn prepare(&self, user: u64, text: &str) { + self.did.borrow_mut().push(Did::Prepare(user, text.into())); + } fn ask_addressed(&self, id: &str, user: u64, _text: &str) -> bool { self.did.borrow_mut().push(Did::Addressed(id.into(), user)); true @@ -553,6 +579,29 @@ mod tests { assert!(f.take().is_empty(), "missed while busy, and a wake word alone waits silently"); } + fn partial(speaker: u64, text: &str) -> CoreEvent { + CoreEvent::Partial(Heard { speaker_id: speaker, speaker_name: "x".into(), utt: 1 }, text.into()) + } + + #[test] + fn a_partial_that_reads_as_a_call_lets_the_brain_get_ready() { + let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord); + c.on_event(&partial(A, "자비스 오늘"), ms(t, 0)); + c.on_event(&partial(B, "오늘 뭐 먹지"), ms(t, 0)); // not a call + c.on_event(&partial(A, "자비스"), ms(t, 0)); // the name alone: nothing to get ready for + c.on_event(&partial(A, "자비스 멈춰"), ms(t, 0)); // a stop is not answered + assert_eq!(f.take(), vec![Did::Prepare(A, "자비스 오늘".into())]); + // After a wake word said alone, the next utterance's partial is joined to it, as its final. + c.on_event(&fin_utt(B, 1, "자비스"), ms(t, 100)); + c.on_event(&partial(B, "지금 몇 시"), ms(t, 300)); + assert_eq!(f.take(), vec![Did::Prepare(B, "자비스 지금 몇 시".into())]); + // Not while an answer runs. + c.on_event(&fin(A, "자비스 날씨", 100), ms(t, 400)); + f.take(); + c.on_event(&partial(A, "자비스 내일은"), ms(t, 500)); + assert!(f.take().is_empty()); + } + #[test] fn the_earcon_can_be_turned_off() { let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord); diff --git a/app/src/conversation.spec.md b/app/src/conversation.spec.md index f7140b9..7f0dd5d 100644 --- a/app/src/conversation.spec.md +++ b/app/src/conversation.spec.md @@ -169,9 +169,14 @@ never takes the voice turn and never goes through the response policy. `app/src/conversation.rs` runs between the core's events and everything else that reads them (every event is passed on unchanged). It talks to the core through a small `Voice` interface (answer, cancel -one request, stop all, earcon, ask "addressed", ask "rank"), so the glue is tested with a fake. +one request, stop all, earcon, prepare, ask "addressed", ask "rank"), so the glue is tested with a fake. - `Chat` and `ReplyText`: chat calls, above. +- `Partial` (getting ready): a partial transcript that call detection already reads as a request + (joined to the speaker's waiting wake word alone, as its final would be; not the wake word alone, + not a stop) is sent to the brain as `prepare(user, text)`, so the brain judges what the call needs + while the speaker is still talking (`voice_worker.spec.md`, `prepare`). Not while an answer runs: + the call may not be taken, and the brain is busy answering. - `SpeechStart` opens the speaker's utterance; `SpeakingStopped` marks it ending. - `Final`: a request gets the next id, `ended_at` = arrival minus the final's `wait_ms`, and goes through call detection. Ignore closes the utterance. Stop stops all speech, runs the policy's stop, @@ -202,7 +207,8 @@ modes) and every policy rule above with fake times (settling, ordering by end ti interruption, ranking tie, ranking timeout, stop dropping the queue), and the conversation loop with a fake voice side (answering and queueing, a busy drop recorded without text, voice stop, addressed yes, no and timeout, the earcon before each answer, once for a wake word alone, none while busy or when -off, interruption cancelling the right speech, a ranking deciding, end times from +off, partials that read as a call sent to get ready (joined to a waiting wake word, none while +answering), interruption cancelling the right speech, a ranking deciding, end times from `wait_ms`, and chat: an answer posted as a reply once, per-channel switches, no answers to bots or to a stop, chat wake words and designated people, a failed answer posting nothing). `tests/worker_protocol.rs` checks an answer spoken under its id and both judgements against the real Python worker with a fake brain. diff --git a/app/src/discord.rs b/app/src/discord.rs index 8bbe923..7a2b651 100644 --- a/app/src/discord.rs +++ b/app/src/discord.rs @@ -441,6 +441,11 @@ impl DiscordCore { self.inner.worker.addressed(id, user, text) } + /// Tell the brain a call from `user` is being said; nothing comes back. + pub fn prepare(&self, user: u64, text: &str) { + self.inner.worker.prepare(user, text); + } + /// Ask the brain to rank candidates `(id, user, text)`; the answer comes as `CoreEvent::Ranked`. pub fn ask_rank(&self, id: &str, candidates: &[(String, u64, String)]) -> bool { self.inner.worker.rank(id, candidates) diff --git a/app/src/discord_core.spec.md b/app/src/discord_core.spec.md index b145ae3..bcd5252 100644 --- a/app/src/discord_core.spec.md +++ b/app/src/discord_core.spec.md @@ -155,7 +155,8 @@ and the late "left" update cancelled the new join (seen on real Discord). from the brain, a refused `reply` (`brain_unavailable`) or a lost worker ends it as failed. `cancel(id)` stops one request (its clip if playing, the worker's unsent sentences, or the speech of an answer still being written) and the rest of the queue goes on. `ask_addressed(id, user, text)` and - `ask_rank(id, candidates)` pass the conversation's questions to the brain. + `ask_rank(id, candidates)` pass the conversation's questions to the brain. `prepare(user, text)` sends `prepare` (a + call still being said) and expects nothing back. - **Earcon** (M4): `earcon()` queues the app's own earcon clip (`app/src/earcon.rs`, 48 kHz stereo, peak 0.3 of full scale) as a request of its own: no worker round trip, `SpeakStart`/`SpeakEnd` like any request, dropped by a stop. None when not in a voice channel. diff --git a/app/src/worker_client.rs b/app/src/worker_client.rs index 79e4e19..633b4ba 100644 --- a/app/src/worker_client.rs +++ b/app/src/worker_client.rs @@ -343,6 +343,11 @@ impl WorkerClient { self.send_json(json!({"type": "addressed", "id": id, "user": user.to_string(), "text": text})) } + /// Tell the brain a call is being said (a partial transcript), so it gets ready to answer it. + pub fn prepare(&self, user: u64, text: &str) -> bool { + self.send_json(json!({"type": "prepare", "user": user.to_string(), "text": text})) + } + /// Ask the brain to rank tied candidates `(id, user, text)` by importance. pub fn rank(&self, id: &str, candidates: &[(String, u64, String)]) -> bool { let list: Vec = candidates.iter() diff --git a/app/tests/live_round_trip.rs b/app/tests/live_round_trip.rs index 9a50e34..3877064 100644 --- a/app/tests/live_round_trip.rs +++ b/app/tests/live_round_trip.rs @@ -72,7 +72,8 @@ struct Ears { jarvis: u64, ssrc: Arc>>, stream: Arc>>>, - first_audible: Arc>>, + /// When each audible 20 ms frame arrived (the earcon first, then the answer). + audible: Arc>>, } #[async_trait] @@ -95,7 +96,7 @@ impl VoiceHandler for Ears { // Audible: louder than -36 dBFS. Silence frames do not count. let rms = (f.iter().map(|&x| f64::from(x).powi(2)).sum::() / f.len().max(1) as f64).sqrt(); if rms > 500.0 { - self.first_audible.lock().unwrap().get_or_insert_with(Instant::now); + self.audible.lock().unwrap().push(Instant::now()); } } } @@ -261,7 +262,7 @@ async fn conversation_round_trip_on_real_discord() { // The previous answer is over before the next question. until("Jarvis to finish speaking", 60_000, || !core.speaking()).await; let before = events.lock().unwrap().len(); - *ears.first_audible.lock().unwrap() = None; + ears.audible.lock().unwrap().clear(); let track = call.lock().await.play_only_input(Input::from(wav.clone())); let deadline = Instant::now() + Duration::from_secs(5); let started = loop { @@ -305,8 +306,18 @@ async fn conversation_round_trip_on_real_discord() { text: finals.iter().map(|(_, f)| f.text.as_str()).collect::>().join(" | "), stt_ms: f.stt_ms, })); - let speak_start = ev.iter().find_map(|(t, e)| matches!(e, CoreEvent::SpeakStart(_)).then_some(*t)); - let first_sound = *ears.first_audible.lock().unwrap(); + // The first sound is the earcon; the answer's own sound starts after its SpeakStart. + let reply_id = ev.iter().find_map(|(_, e)| match e { + CoreEvent::ReplyText { id, .. } => Some(id.clone()), + _ => None, + }); + let speak_start = ev.iter().find_map(|(t, e)| match e { + CoreEvent::SpeakStart(id) if Some(id) == reply_id.as_ref() => Some(*t), + _ => None, + }); + let audible = ears.audible.lock().unwrap().clone(); + let first_sound = audible.first().copied(); + let answer_sound = speak_start.and_then(|s| audible.iter().copied().find(|t| *t >= s)); rows.push(json!({ "round": round, "question": text, "heard": final_ev.as_ref().map(|(_, f)| f.text.clone()), @@ -316,6 +327,7 @@ async fn conversation_round_trip_on_real_discord() { "answer_text_ms": ms(reply.as_ref().map(|r| r.0)), "speak_start_ms": ms(speak_start), "first_sound_ms": ms(first_sound), + "answer_sound_ms": ms(answer_sound), "spoken": finished, })); println!("{phase} {}", rows.last().unwrap()); @@ -324,12 +336,17 @@ async fn conversation_round_trip_on_real_discord() { } stop.store(true, Ordering::Relaxed); let load_finals = match load { Some(h) => Some(h.await.unwrap()), None => None }; - let mut sound: Vec = rows.iter().filter_map(|r| r["first_sound_ms"].as_i64()).collect(); - sound.sort_unstable(); - let pick = |q: f64| sound.get(((sound.len() as f64 - 1.0) * q).round() as usize).copied(); + let stats = |key: &str| { + let mut v: Vec = rows.iter().filter_map(|r| r[key].as_i64()).collect(); + v.sort_unstable(); + let pick = |q: f64| v.get(((v.len() as f64 - 1.0) * q).round() as usize).copied(); + (v.len(), json!({"min": v.first(), "median": pick(0.5), "p90": pick(0.9), "max": v.last()})) + }; + let (answered, answer_sound) = stats("answer_sound_ms"); report.insert(phase.into(), json!({ - "rows": rows, "answered": sound.len(), "asked": rows.len(), - "first_sound_ms": {"min": sound.first(), "median": pick(0.5), "p90": pick(0.9), "max": sound.last()}, + "rows": rows, "answered": answered, "asked": rows.len(), + "first_sound_ms": stats("first_sound_ms").1, + "answer_sound_ms": answer_sound, "background_finals": load_finals, })); }