//! The bot's mouth: turns text into queued speech, one request after another. //! //! Each request goes to the voice worker as TTS; the worker sends back one PCM chunk per sentence. //! Chunks play in order as soon as they arrive, so the first sentence plays while later ones are still //! being made. Requests play first come, first served. `stop()` silences everything at once and //! cancels what the worker has not made yet. use std::collections::{BTreeMap, VecDeque}; use crate::worker_client::TtsDone; pub trait TtsBackend { /// Send text for request `id`; false when the worker is not connected (the request then fails). fn tts(&self, id: &str, text: &str, more: bool) -> bool; fn tts_cancel(&self, id: &str); /// Request `id` ended after `played` of its sentences started playing, the last of them for /// `last_ms` before it was stopped (None: played to its end). The worker's recording keeps that /// much (voice_worker.spec.md, "Recordings"). fn tts_played(&self, id: &str, played: u32, cancelled: bool, last_ms: Option); } /// Plays one PCM clip at a time; its owner calls `Playback::clip_ended()` when the clip finishes. pub trait ClipSink { fn play(&mut self, pcm: Vec); fn stop(&mut self); } #[derive(Debug, Clone, PartialEq, Eq)] pub struct PlaybackEnd { pub id: String, /// Sentences that were played. pub played: u32, pub cancelled: bool, pub failed: bool, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum PlaybackEvent { Start(String), End(PlaybackEnd), } struct Request { id: String, chunks: BTreeMap>, next: u32, done: bool, failed: bool, played: u32, started: bool, } pub struct Playback { backend: B, sink: S, queue: VecDeque, playing: bool, counter: u64, events: Vec, /// When the clip now playing started. clip_started: Option, } impl Playback { pub fn new(backend: B, sink: S) -> Self { Self { backend, sink, queue: VecDeque::new(), playing: false, counter: 0, events: vec![], clip_started: None } } /// True while a clip is playing or a request is waiting for its audio. pub fn busy(&self) -> bool { self.playing || !self.queue.is_empty() } /// Start and end events since the last call. pub fn take_events(&mut self) -> Vec { std::mem::take(&mut self.events) } /// Queue `text` as a new request; with `more`, add text later with `continue_text`. Returns its id. pub fn say(&mut self, text: &str, more: bool) -> String { self.counter += 1; let id = format!("r{}", self.counter); self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false, played: 0, started: false }); crate::debug_log!("play", "say id={id} chars={} more={more} queued={}", text.chars().count(), self.queue.len()); self.backend.tts(&id, text, more); id } /// Queue a request whose speech the worker makes from the brain's answer (`reply` with `speak`), /// under the returned id. Nothing is sent here. pub fn adopt(&mut self) -> String { self.counter += 1; let id = format!("r{}", self.counter); self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false, played: 0, started: false }); crate::debug_log!("play", "answer id={id} queued={}", self.queue.len()); id } /// Queue a clip the app already has (the earcon) as a request of its own. Returns its id. pub fn clip(&mut self, pcm: Vec) -> String { self.counter += 1; let id = format!("r{}", self.counter); self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::from([(0, pcm)]), next: 0, done: true, failed: false, played: 0, started: false }); crate::debug_log!("play", "clip id={id} queued={}", self.queue.len()); self.pump(); id } /// End a request that will get no audio (the brain gave no answer). pub fn fail(&mut self, id: &str) { if let Some(r) = self.queue.iter_mut().find(|r| r.id == id && !r.done) { r.done = true; r.failed = true; self.pump(); } } /// Stop one request: its clip if it is playing, its unsent sentences at the worker. The rest of the /// queue goes on. pub fn cancel(&mut self, id: &str) { let Some(pos) = self.queue.iter().position(|r| r.id == id) else { return }; let r = self.queue.remove(pos).expect("found"); if !r.done { self.backend.tts_cancel(&r.id); } let mut last_ms = None; if pos == 0 && self.playing { self.playing = false; last_ms = self.clip_elapsed_ms(); self.sink.stop(); } crate::debug_log!("play", "cancel id={} played={}", r.id, r.played); self.ended(PlaybackEnd { id: r.id, played: r.played, cancelled: true, failed: false }, last_ms); self.pump(); } /// More text for an open request (an LLM answer that is still streaming). pub fn continue_text(&mut self, id: &str, text: &str, more: bool) { if self.queue.iter().any(|r| r.id == id && !r.done) { self.backend.tts(id, text, more); } } /// A sentence of audio arrived from the worker. pub fn audio(&mut self, id: &str, seq: u32, pcm: Vec) { if let Some(r) = self.queue.iter_mut().find(|r| r.id == id) { r.chunks.insert(seq, pcm); self.pump(); } } /// The worker finished a request (all sentences sent, cancelled, or failed). pub fn done(&mut self, d: &TtsDone) { if let Some(r) = self.queue.iter_mut().find(|r| r.id == d.id) { r.done = true; r.failed = d.failed; self.pump(); } } /// The sink finished playing the current clip. pub fn clip_ended(&mut self) { self.playing = false; self.pump(); } /// Silence now: stop the clip, drop every queued request, cancel the worker's work. pub fn stop(&mut self) -> usize { let stopped: Vec = self.queue.drain(..).collect(); let was_playing = self.playing; let head_ms = if was_playing { self.clip_elapsed_ms() } else { None }; self.playing = false; for r in stopped.iter().filter(|r| !r.done) { self.backend.tts_cancel(&r.id); } if was_playing { self.sink.stop(); } for (i, r) in stopped.iter().enumerate() { let last_ms = if i == 0 { head_ms } else { None }; self.ended(PlaybackEnd { id: r.id.clone(), played: r.played, cancelled: true, failed: false }, last_ms); } crate::debug_log!("play", "stop requests={} was_playing={was_playing}", stopped.len()); stopped.len() } /// A request ended: tell the worker how much of it was played, then the owner. fn ended(&mut self, end: PlaybackEnd, last_ms: Option) { self.backend.tts_played(&end.id, end.played, end.cancelled, last_ms); self.events.push(PlaybackEvent::End(end)); } /// How long the clip now playing has played. fn clip_elapsed_ms(&self) -> Option { self.clip_started.map(|t| t.elapsed().as_millis().min(u32::MAX as u128) as u32) } fn pump(&mut self) { while !self.playing { let Some(head) = self.queue.front_mut() else { return }; if !head.chunks.contains_key(&head.next) && head.done { // A sentence failed: skip its gap and play what did arrive. if let Some(&first) = head.chunks.keys().next() { head.next = first; } } if let Some(pcm) = head.chunks.remove(&head.next) { head.next += 1; head.played += 1; if !head.started { head.started = true; self.events.push(PlaybackEvent::Start(head.id.clone())); } self.playing = true; self.clip_started = Some(std::time::Instant::now()); self.sink.play(pcm); return; } if !head.done { return; } let r = self.queue.pop_front().expect("head exists"); crate::debug_log!("play", "said id={} sentences={} failed={}", r.id, r.played, r.failed); self.ended(PlaybackEnd { id: r.id, played: r.played, cancelled: false, failed: r.failed }, None); } } } #[cfg(test)] mod tests { use super::*; use std::{cell::RefCell, rc::Rc}; #[derive(Default)] struct Log { sent: Vec<(String, String, bool)>, cancelled: Vec, played: Vec, stops: usize, reported: Vec<(String, u32, bool, Option)>, } #[derive(Clone, Default)] struct Fake(Rc>); impl TtsBackend for Fake { fn tts(&self, id: &str, text: &str, more: bool) -> bool { self.0.borrow_mut().sent.push((id.into(), text.into(), more)); true } fn tts_cancel(&self, id: &str) { self.0.borrow_mut().cancelled.push(id.into()); } fn tts_played(&self, id: &str, played: u32, cancelled: bool, last_ms: Option) { self.0.borrow_mut().reported.push((id.into(), played, cancelled, last_ms)); } } impl ClipSink for Fake { fn play(&mut self, pcm: Vec) { self.0.borrow_mut().played.push(String::from_utf8(pcm).unwrap()); } fn stop(&mut self) { self.0.borrow_mut().stops += 1; } } fn setup() -> (Playback, Fake) { let f = Fake::default(); (Playback::new(f.clone(), f.clone()), f) } fn done(id: &str, failed: bool) -> TtsDone { TtsDone { id: id.into(), chunks: 0, cancelled: false, failed } } fn end(id: &str, played: u32, cancelled: bool, failed: bool) -> PlaybackEvent { PlaybackEvent::End(PlaybackEnd { id: id.into(), played, cancelled, failed }) } fn b(s: &str) -> Vec { s.as_bytes().to_vec() } #[test] fn plays_the_first_sentence_on_arrival_and_the_next_when_the_clip_ends() { let (mut p, f) = setup(); let id = p.say("하나. 둘.", false); assert_eq!(f.0.borrow().sent, vec![(id.clone(), "하나. 둘.".into(), false)]); p.audio(&id, 0, b("s0")); assert_eq!(f.0.borrow().played, vec!["s0"]); p.audio(&id, 1, b("s1")); assert_eq!(f.0.borrow().played, vec!["s0"]); p.clip_ended(); assert_eq!(f.0.borrow().played, vec!["s0", "s1"]); p.done(&done(&id, false)); assert_eq!(p.take_events(), vec![PlaybackEvent::Start(id.clone())]); p.clip_ended(); assert_eq!(p.take_events(), vec![end(&id, 2, false, false)]); assert!(!p.busy()); } #[test] fn the_worker_is_told_how_much_of_each_request_was_played() { let (mut p, f) = setup(); let a = p.say("하나. 둘.", false); p.audio(&a, 0, b("a0")); p.audio(&a, 1, b("a1")); p.done(&done(&a, false)); p.clip_ended(); p.clip_ended(); let c = p.say("셋. 넷.", false); p.audio(&c, 0, b("c0")); p.audio(&c, 1, b("c1")); p.done(&done(&c, false)); std::thread::sleep(std::time::Duration::from_millis(60)); p.cancel(&c); // stopped about 60 ms into its first sentence let e = p.say("여섯.", false); p.audio(&e, 0, b("e0")); let d = p.say("다섯.", false); p.stop(); // e stopped at once, d before any sound let r = f.0.borrow().reported.clone(); assert_eq!(r[0], (a, 2, false, None), "played to its end"); let (id, n, cancelled, ms) = r[1].clone(); assert_eq!((id, n, cancelled), (c, 1, true)); assert!(ms.is_some_and(|ms| (50..1000).contains(&ms)), "how long the stopped sentence played: {ms:?}"); assert_eq!((r[2].0.clone(), r[2].1, r[2].2), (e, 1, true)); assert!(r[2].3.is_some_and(|ms| ms < 50)); assert_eq!(r[3], (d, 0, true, None)); } #[test] fn requests_play_in_the_order_they_were_made() { let (mut p, f) = setup(); let a = p.say("a", false); let c = p.say("b", false); p.audio(&c, 0, b("b0")); assert!(f.0.borrow().played.is_empty()); p.audio(&a, 0, b("a0")); p.done(&done(&a, false)); p.done(&done(&c, false)); p.clip_ended(); p.clip_ended(); assert_eq!(f.0.borrow().played, vec!["a0", "b0"]); let ends: Vec<_> = p.take_events().into_iter().filter_map(|e| match e { PlaybackEvent::End(e) => Some(e.id), _ => None }).collect(); assert_eq!(ends, vec![a, c]); } #[test] fn sentences_stay_in_order_when_they_arrive_out_of_order() { let (mut p, f) = setup(); let id = p.say("x", true); p.audio(&id, 1, b("s1")); assert!(f.0.borrow().played.is_empty()); p.audio(&id, 0, b("s0")); p.clip_ended(); assert_eq!(f.0.borrow().played, vec!["s0", "s1"]); } #[test] fn a_failed_sentence_is_skipped_and_the_rest_still_play() { let (mut p, f) = setup(); let id = p.say("a. b. c.", false); p.audio(&id, 0, b("s0")); p.audio(&id, 2, b("s2")); p.done(&done(&id, true)); p.clip_ended(); p.clip_ended(); assert_eq!(f.0.borrow().played, vec!["s0", "s2"]); assert_eq!(p.take_events().last(), Some(&end(&id, 2, false, true))); } #[test] fn stop_silences_cancels_unfinished_and_drops_the_queue() { let (mut p, f) = setup(); let a = p.say("a", true); let c = p.say("b", false); p.audio(&a, 0, b("a0")); p.done(&done(&c, false)); p.take_events(); assert_eq!(p.stop(), 2); assert_eq!(f.0.borrow().stops, 1); assert_eq!(f.0.borrow().cancelled, vec![a.clone()]); assert_eq!(p.take_events(), vec![end(&a, 1, true, false), end(&c, 0, true, false)]); p.audio(&a, 1, b("a1")); p.clip_ended(); assert_eq!(f.0.borrow().played, vec!["a0"]); assert!(!p.busy()); } #[test] fn stop_with_nothing_to_say_touches_nothing() { let (mut p, f) = setup(); assert_eq!(p.stop(), 0); assert_eq!(f.0.borrow().stops, 0); } #[test] fn text_can_be_added_to_an_open_request_but_not_a_finished_one() { let (mut p, f) = setup(); let id = p.say("첫 문장.", true); p.continue_text(&id, " 둘째 문장.", false); p.done(&done(&id, false)); p.continue_text(&id, "늦은 글", false); let sent: Vec<_> = f.0.borrow().sent.iter().map(|(_, t, m)| (t.clone(), *m)).collect(); assert_eq!(sent, vec![("첫 문장.".into(), true), (" 둘째 문장.".into(), false)]); } #[test] fn an_adopted_answer_plays_the_brains_audio_without_sending_text() { let (mut p, f) = setup(); let id = p.adopt(); assert!(f.0.borrow().sent.is_empty(), "the brain speaks it under this id"); p.audio(&id, 0, b("a0")); p.done(&done(&id, false)); p.clip_ended(); assert_eq!(f.0.borrow().played, vec!["a0"]); assert_eq!(p.take_events(), vec![PlaybackEvent::Start(id.clone()), end(&id, 1, false, false)]); assert_ne!(p.adopt(), id, "ids never repeat"); } #[test] fn an_answer_with_no_text_fails_and_the_next_request_plays() { let (mut p, f) = setup(); let id = p.adopt(); let c = p.say("b", false); p.audio(&c, 0, b("b0")); assert!(f.0.borrow().played.is_empty()); p.fail(&id); assert_eq!(f.0.borrow().played, vec!["b0"]); assert_eq!(p.take_events()[0], end(&id, 0, false, true)); } #[test] fn cancel_stops_one_request_and_leaves_the_others() { let (mut p, f) = setup(); let a = p.say("a", true); let c = p.say("b", false); p.audio(&a, 0, b("a0")); p.audio(&c, 0, b("b0")); p.take_events(); p.cancel(&a); assert_eq!(f.0.borrow().stops, 1); assert_eq!(f.0.borrow().cancelled, vec![a.clone()]); assert_eq!(f.0.borrow().played, vec!["a0", "b0"], "the next request plays at once"); assert_eq!(p.take_events(), vec![end(&a, 1, true, false), PlaybackEvent::Start(c.clone())]); // A queued request that has not started is cancelled without touching the clip. let d = p.say("d", false); p.cancel(&d); assert_eq!(f.0.borrow().stops, 1); assert_eq!(p.take_events(), vec![end(&d, 0, true, false)]); p.cancel("nope"); assert!(p.take_events().is_empty()); } #[test] fn a_request_the_worker_never_accepted_fails_without_blocking_the_next() { let (mut p, f) = setup(); let a = p.say("a", false); let c = p.say("b", false); p.done(&done(&a, true)); p.audio(&c, 0, b("b0")); assert_eq!(f.0.borrow().played, vec!["b0"]); assert_eq!(p.take_events()[0], end(&a, 0, false, true)); } #[test] fn a_clip_plays_at_once_then_the_answer_queued_after_it() { let (mut p, f) = setup(); let cue = p.clip(b("ding")); let id = p.adopt(); assert_eq!(f.0.borrow().played, vec!["ding"], "the clip needs no worker"); assert!(f.0.borrow().sent.is_empty()); p.audio(&id, 0, b("a0")); assert_eq!(f.0.borrow().played, vec!["ding"]); p.clip_ended(); assert_eq!(f.0.borrow().played, vec!["ding", "a0"]); assert_eq!(p.take_events(), vec![PlaybackEvent::Start(cue.clone()), end(&cue, 1, false, false), PlaybackEvent::Start(id.clone())]); // A stop drops a clip still waiting its turn, without asking the worker to cancel it. let cue2 = p.clip(b("ding")); p.stop(); assert_eq!(f.0.borrow().cancelled, vec![id.clone()]); assert!(p.take_events().contains(&end(&cue2, 0, true, false))); } }