//! The bot's mouth: turns text into queued speech, one request after another. //! //! Each request goes to the voice worker as TTS; the worker sends back one PCM chunk per sentence. //! Chunks play in order as soon as they arrive, so the first sentence plays while later ones are still //! being made. Requests play first come, first served. `stop()` silences everything at once and //! cancels what the worker has not made yet. use std::collections::{BTreeMap, VecDeque}; use crate::worker_client::TtsDone; pub trait TtsBackend { /// Send text for request `id`; false when the worker is not connected (the request then fails). fn tts(&self, id: &str, text: &str, more: bool) -> bool; fn tts_cancel(&self, id: &str); } /// Plays one PCM clip at a time; its owner calls `Playback::clip_ended()` when the clip finishes. pub trait ClipSink { fn play(&mut self, pcm: Vec); fn stop(&mut self); } #[derive(Debug, Clone, PartialEq, Eq)] pub struct PlaybackEnd { pub id: String, /// Sentences that were played. pub played: u32, pub cancelled: bool, pub failed: bool, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum PlaybackEvent { Start(String), End(PlaybackEnd), } struct Request { id: String, chunks: BTreeMap>, next: u32, done: bool, failed: bool, played: u32, started: bool, } pub struct Playback { backend: B, sink: S, queue: VecDeque, playing: bool, counter: u64, events: Vec, } impl Playback { pub fn new(backend: B, sink: S) -> Self { Self { backend, sink, queue: VecDeque::new(), playing: false, counter: 0, events: vec![] } } /// True while a clip is playing or a request is waiting for its audio. pub fn busy(&self) -> bool { self.playing || !self.queue.is_empty() } /// Start and end events since the last call. pub fn take_events(&mut self) -> Vec { std::mem::take(&mut self.events) } /// Queue `text` as a new request; with `more`, add text later with `continue_text`. Returns its id. pub fn say(&mut self, text: &str, more: bool) -> String { self.counter += 1; let id = format!("r{}", self.counter); self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false, played: 0, started: false }); crate::debug_log!("play", "say id={id} chars={} more={more} queued={}", text.chars().count(), self.queue.len()); self.backend.tts(&id, text, more); id } /// Queue a request whose speech the worker makes from the brain's answer (`reply` with `speak`), /// under the returned id. Nothing is sent here. pub fn adopt(&mut self) -> String { self.counter += 1; let id = format!("r{}", self.counter); self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false, played: 0, started: false }); crate::debug_log!("play", "answer id={id} queued={}", self.queue.len()); id } /// Queue a clip the app already has (the earcon) as a request of its own. Returns its id. pub fn clip(&mut self, pcm: Vec) -> String { self.counter += 1; let id = format!("r{}", self.counter); self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::from([(0, pcm)]), next: 0, done: true, failed: false, played: 0, started: false }); crate::debug_log!("play", "clip id={id} queued={}", self.queue.len()); self.pump(); id } /// End a request that will get no audio (the brain gave no answer). pub fn fail(&mut self, id: &str) { if let Some(r) = self.queue.iter_mut().find(|r| r.id == id && !r.done) { r.done = true; r.failed = true; self.pump(); } } /// Stop one request: its clip if it is playing, its unsent sentences at the worker. The rest of the /// queue goes on. pub fn cancel(&mut self, id: &str) { let Some(pos) = self.queue.iter().position(|r| r.id == id) else { return }; let r = self.queue.remove(pos).expect("found"); if !r.done { self.backend.tts_cancel(&r.id); } if pos == 0 && self.playing { self.playing = false; self.sink.stop(); } crate::debug_log!("play", "cancel id={} played={}", r.id, r.played); self.events.push(PlaybackEvent::End(PlaybackEnd { id: r.id, played: r.played, cancelled: true, failed: false })); self.pump(); } /// More text for an open request (an LLM answer that is still streaming). pub fn continue_text(&mut self, id: &str, text: &str, more: bool) { if self.queue.iter().any(|r| r.id == id && !r.done) { self.backend.tts(id, text, more); } } /// A sentence of audio arrived from the worker. pub fn audio(&mut self, id: &str, seq: u32, pcm: Vec) { if let Some(r) = self.queue.iter_mut().find(|r| r.id == id) { r.chunks.insert(seq, pcm); self.pump(); } } /// The worker finished a request (all sentences sent, cancelled, or failed). pub fn done(&mut self, d: &TtsDone) { if let Some(r) = self.queue.iter_mut().find(|r| r.id == d.id) { r.done = true; r.failed = d.failed; self.pump(); } } /// The sink finished playing the current clip. pub fn clip_ended(&mut self) { self.playing = false; self.pump(); } /// Silence now: stop the clip, drop every queued request, cancel the worker's work. pub fn stop(&mut self) -> usize { let stopped: Vec = self.queue.drain(..).collect(); let was_playing = self.playing; self.playing = false; for r in stopped.iter().filter(|r| !r.done) { self.backend.tts_cancel(&r.id); } if was_playing { self.sink.stop(); } for r in &stopped { self.events.push(PlaybackEvent::End(PlaybackEnd { id: r.id.clone(), played: r.played, cancelled: true, failed: false })); } crate::debug_log!("play", "stop requests={} was_playing={was_playing}", stopped.len()); stopped.len() } fn pump(&mut self) { while !self.playing { let Some(head) = self.queue.front_mut() else { return }; if !head.chunks.contains_key(&head.next) && head.done { // A sentence failed: skip its gap and play what did arrive. if let Some(&first) = head.chunks.keys().next() { head.next = first; } } if let Some(pcm) = head.chunks.remove(&head.next) { head.next += 1; head.played += 1; if !head.started { head.started = true; self.events.push(PlaybackEvent::Start(head.id.clone())); } self.playing = true; self.sink.play(pcm); return; } if !head.done { return; } let r = self.queue.pop_front().expect("head exists"); crate::debug_log!("play", "said id={} sentences={} failed={}", r.id, r.played, r.failed); self.events.push(PlaybackEvent::End(PlaybackEnd { id: r.id, played: r.played, cancelled: false, failed: r.failed })); } } } #[cfg(test)] mod tests { use super::*; use std::{cell::RefCell, rc::Rc}; #[derive(Default)] struct Log { sent: Vec<(String, String, bool)>, cancelled: Vec, played: Vec, stops: usize, } #[derive(Clone, Default)] struct Fake(Rc>); impl TtsBackend for Fake { fn tts(&self, id: &str, text: &str, more: bool) -> bool { self.0.borrow_mut().sent.push((id.into(), text.into(), more)); true } fn tts_cancel(&self, id: &str) { self.0.borrow_mut().cancelled.push(id.into()); } } impl ClipSink for Fake { fn play(&mut self, pcm: Vec) { self.0.borrow_mut().played.push(String::from_utf8(pcm).unwrap()); } fn stop(&mut self) { self.0.borrow_mut().stops += 1; } } fn setup() -> (Playback, Fake) { let f = Fake::default(); (Playback::new(f.clone(), f.clone()), f) } fn done(id: &str, failed: bool) -> TtsDone { TtsDone { id: id.into(), chunks: 0, cancelled: false, failed } } fn end(id: &str, played: u32, cancelled: bool, failed: bool) -> PlaybackEvent { PlaybackEvent::End(PlaybackEnd { id: id.into(), played, cancelled, failed }) } fn b(s: &str) -> Vec { s.as_bytes().to_vec() } #[test] fn plays_the_first_sentence_on_arrival_and_the_next_when_the_clip_ends() { let (mut p, f) = setup(); let id = p.say("하나. 둘.", false); assert_eq!(f.0.borrow().sent, vec![(id.clone(), "하나. 둘.".into(), false)]); p.audio(&id, 0, b("s0")); assert_eq!(f.0.borrow().played, vec!["s0"]); p.audio(&id, 1, b("s1")); assert_eq!(f.0.borrow().played, vec!["s0"]); p.clip_ended(); assert_eq!(f.0.borrow().played, vec!["s0", "s1"]); p.done(&done(&id, false)); assert_eq!(p.take_events(), vec![PlaybackEvent::Start(id.clone())]); p.clip_ended(); assert_eq!(p.take_events(), vec![end(&id, 2, false, false)]); assert!(!p.busy()); } #[test] fn requests_play_in_the_order_they_were_made() { let (mut p, f) = setup(); let a = p.say("a", false); let c = p.say("b", false); p.audio(&c, 0, b("b0")); assert!(f.0.borrow().played.is_empty()); p.audio(&a, 0, b("a0")); p.done(&done(&a, false)); p.done(&done(&c, false)); p.clip_ended(); p.clip_ended(); assert_eq!(f.0.borrow().played, vec!["a0", "b0"]); let ends: Vec<_> = p.take_events().into_iter().filter_map(|e| match e { PlaybackEvent::End(e) => Some(e.id), _ => None }).collect(); assert_eq!(ends, vec![a, c]); } #[test] fn sentences_stay_in_order_when_they_arrive_out_of_order() { let (mut p, f) = setup(); let id = p.say("x", true); p.audio(&id, 1, b("s1")); assert!(f.0.borrow().played.is_empty()); p.audio(&id, 0, b("s0")); p.clip_ended(); assert_eq!(f.0.borrow().played, vec!["s0", "s1"]); } #[test] fn a_failed_sentence_is_skipped_and_the_rest_still_play() { let (mut p, f) = setup(); let id = p.say("a. b. c.", false); p.audio(&id, 0, b("s0")); p.audio(&id, 2, b("s2")); p.done(&done(&id, true)); p.clip_ended(); p.clip_ended(); assert_eq!(f.0.borrow().played, vec!["s0", "s2"]); assert_eq!(p.take_events().last(), Some(&end(&id, 2, false, true))); } #[test] fn stop_silences_cancels_unfinished_and_drops_the_queue() { let (mut p, f) = setup(); let a = p.say("a", true); let c = p.say("b", false); p.audio(&a, 0, b("a0")); p.done(&done(&c, false)); p.take_events(); assert_eq!(p.stop(), 2); assert_eq!(f.0.borrow().stops, 1); assert_eq!(f.0.borrow().cancelled, vec![a.clone()]); assert_eq!(p.take_events(), vec![end(&a, 1, true, false), end(&c, 0, true, false)]); p.audio(&a, 1, b("a1")); p.clip_ended(); assert_eq!(f.0.borrow().played, vec!["a0"]); assert!(!p.busy()); } #[test] fn stop_with_nothing_to_say_touches_nothing() { let (mut p, f) = setup(); assert_eq!(p.stop(), 0); assert_eq!(f.0.borrow().stops, 0); } #[test] fn text_can_be_added_to_an_open_request_but_not_a_finished_one() { let (mut p, f) = setup(); let id = p.say("첫 문장.", true); p.continue_text(&id, " 둘째 문장.", false); p.done(&done(&id, false)); p.continue_text(&id, "늦은 글", false); let sent: Vec<_> = f.0.borrow().sent.iter().map(|(_, t, m)| (t.clone(), *m)).collect(); assert_eq!(sent, vec![("첫 문장.".into(), true), (" 둘째 문장.".into(), false)]); } #[test] fn an_adopted_answer_plays_the_brains_audio_without_sending_text() { let (mut p, f) = setup(); let id = p.adopt(); assert!(f.0.borrow().sent.is_empty(), "the brain speaks it under this id"); p.audio(&id, 0, b("a0")); p.done(&done(&id, false)); p.clip_ended(); assert_eq!(f.0.borrow().played, vec!["a0"]); assert_eq!(p.take_events(), vec![PlaybackEvent::Start(id.clone()), end(&id, 1, false, false)]); assert_ne!(p.adopt(), id, "ids never repeat"); } #[test] fn an_answer_with_no_text_fails_and_the_next_request_plays() { let (mut p, f) = setup(); let id = p.adopt(); let c = p.say("b", false); p.audio(&c, 0, b("b0")); assert!(f.0.borrow().played.is_empty()); p.fail(&id); assert_eq!(f.0.borrow().played, vec!["b0"]); assert_eq!(p.take_events()[0], end(&id, 0, false, true)); } #[test] fn cancel_stops_one_request_and_leaves_the_others() { let (mut p, f) = setup(); let a = p.say("a", true); let c = p.say("b", false); p.audio(&a, 0, b("a0")); p.audio(&c, 0, b("b0")); p.take_events(); p.cancel(&a); assert_eq!(f.0.borrow().stops, 1); assert_eq!(f.0.borrow().cancelled, vec![a.clone()]); assert_eq!(f.0.borrow().played, vec!["a0", "b0"], "the next request plays at once"); assert_eq!(p.take_events(), vec![end(&a, 1, true, false), PlaybackEvent::Start(c.clone())]); // A queued request that has not started is cancelled without touching the clip. let d = p.say("d", false); p.cancel(&d); assert_eq!(f.0.borrow().stops, 1); assert_eq!(p.take_events(), vec![end(&d, 0, true, false)]); p.cancel("nope"); assert!(p.take_events().is_empty()); } #[test] fn a_request_the_worker_never_accepted_fails_without_blocking_the_next() { let (mut p, f) = setup(); let a = p.say("a", false); let c = p.say("b", false); p.done(&done(&a, true)); p.audio(&c, 0, b("b0")); assert_eq!(f.0.borrow().played, vec!["b0"]); assert_eq!(p.take_events()[0], end(&a, 0, false, true)); } #[test] fn a_clip_plays_at_once_then_the_answer_queued_after_it() { let (mut p, f) = setup(); let cue = p.clip(b("ding")); let id = p.adopt(); assert_eq!(f.0.borrow().played, vec!["ding"], "the clip needs no worker"); assert!(f.0.borrow().sent.is_empty()); p.audio(&id, 0, b("a0")); assert_eq!(f.0.borrow().played, vec!["ding"]); p.clip_ended(); assert_eq!(f.0.borrow().played, vec!["ding", "a0"]); assert_eq!(p.take_events(), vec![PlaybackEvent::Start(cue.clone()), end(&cue, 1, false, false), PlaybackEvent::Start(id.clone())]); // A stop drops a clip still waiting its turn, without asking the worker to cancel it. let cue2 = p.clip(b("ding")); p.stop(); assert_eq!(f.0.borrow().cancelled, vec![id.clone()]); assert!(p.take_events().contains(&end(&cue2, 0, true, false))); } }