A short two-tone chime, made in code, plays right before each voice answer starts and at once for a wake word said alone while Jarvis is free; the question that joins it gets no second one. call.earcon turns it off. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
432 lines
15 KiB
Rust
432 lines
15 KiB
Rust
//! The bot's mouth: turns text into queued speech, one request after another.
|
|
//!
|
|
//! Each request goes to the voice worker as TTS; the worker sends back one PCM chunk per sentence.
|
|
//! Chunks play in order as soon as they arrive, so the first sentence plays while later ones are still
|
|
//! being made. Requests play first come, first served. `stop()` silences everything at once and
|
|
//! cancels what the worker has not made yet.
|
|
|
|
use std::collections::{BTreeMap, VecDeque};
|
|
|
|
use crate::worker_client::TtsDone;
|
|
|
|
pub trait TtsBackend {
|
|
/// Send text for request `id`; false when the worker is not connected (the request then fails).
|
|
fn tts(&self, id: &str, text: &str, more: bool) -> bool;
|
|
fn tts_cancel(&self, id: &str);
|
|
}
|
|
|
|
/// Plays one PCM clip at a time; its owner calls `Playback::clip_ended()` when the clip finishes.
|
|
pub trait ClipSink {
|
|
fn play(&mut self, pcm: Vec<u8>);
|
|
fn stop(&mut self);
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct PlaybackEnd {
|
|
pub id: String,
|
|
/// Sentences that were played.
|
|
pub played: u32,
|
|
pub cancelled: bool,
|
|
pub failed: bool,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub enum PlaybackEvent {
|
|
Start(String),
|
|
End(PlaybackEnd),
|
|
}
|
|
|
|
struct Request {
|
|
id: String,
|
|
chunks: BTreeMap<u32, Vec<u8>>,
|
|
next: u32,
|
|
done: bool,
|
|
failed: bool,
|
|
played: u32,
|
|
started: bool,
|
|
}
|
|
|
|
pub struct Playback<B: TtsBackend, S: ClipSink> {
|
|
backend: B,
|
|
sink: S,
|
|
queue: VecDeque<Request>,
|
|
playing: bool,
|
|
counter: u64,
|
|
events: Vec<PlaybackEvent>,
|
|
}
|
|
|
|
impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
|
pub fn new(backend: B, sink: S) -> Self {
|
|
Self { backend, sink, queue: VecDeque::new(), playing: false, counter: 0, events: vec![] }
|
|
}
|
|
|
|
/// True while a clip is playing or a request is waiting for its audio.
|
|
pub fn busy(&self) -> bool {
|
|
self.playing || !self.queue.is_empty()
|
|
}
|
|
|
|
/// Start and end events since the last call.
|
|
pub fn take_events(&mut self) -> Vec<PlaybackEvent> {
|
|
std::mem::take(&mut self.events)
|
|
}
|
|
|
|
/// Queue `text` as a new request; with `more`, add text later with `continue_text`. Returns its id.
|
|
pub fn say(&mut self, text: &str, more: bool) -> String {
|
|
self.counter += 1;
|
|
let id = format!("r{}", self.counter);
|
|
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false,
|
|
played: 0, started: false });
|
|
crate::debug_log!("play", "say id={id} chars={} more={more} queued={}", text.chars().count(), self.queue.len());
|
|
self.backend.tts(&id, text, more);
|
|
id
|
|
}
|
|
|
|
/// Queue a request whose speech the worker makes from the brain's answer (`reply` with `speak`),
|
|
/// under the returned id. Nothing is sent here.
|
|
pub fn adopt(&mut self) -> String {
|
|
self.counter += 1;
|
|
let id = format!("r{}", self.counter);
|
|
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false,
|
|
played: 0, started: false });
|
|
crate::debug_log!("play", "answer id={id} queued={}", self.queue.len());
|
|
id
|
|
}
|
|
|
|
/// Queue a clip the app already has (the earcon) as a request of its own. Returns its id.
|
|
pub fn clip(&mut self, pcm: Vec<u8>) -> String {
|
|
self.counter += 1;
|
|
let id = format!("r{}", self.counter);
|
|
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::from([(0, pcm)]), next: 0, done: true,
|
|
failed: false, played: 0, started: false });
|
|
crate::debug_log!("play", "clip id={id} queued={}", self.queue.len());
|
|
self.pump();
|
|
id
|
|
}
|
|
|
|
/// End a request that will get no audio (the brain gave no answer).
|
|
pub fn fail(&mut self, id: &str) {
|
|
if let Some(r) = self.queue.iter_mut().find(|r| r.id == id && !r.done) {
|
|
r.done = true;
|
|
r.failed = true;
|
|
self.pump();
|
|
}
|
|
}
|
|
|
|
/// Stop one request: its clip if it is playing, its unsent sentences at the worker. The rest of the
|
|
/// queue goes on.
|
|
pub fn cancel(&mut self, id: &str) {
|
|
let Some(pos) = self.queue.iter().position(|r| r.id == id) else { return };
|
|
let r = self.queue.remove(pos).expect("found");
|
|
if !r.done {
|
|
self.backend.tts_cancel(&r.id);
|
|
}
|
|
if pos == 0 && self.playing {
|
|
self.playing = false;
|
|
self.sink.stop();
|
|
}
|
|
crate::debug_log!("play", "cancel id={} played={}", r.id, r.played);
|
|
self.events.push(PlaybackEvent::End(PlaybackEnd { id: r.id, played: r.played, cancelled: true,
|
|
failed: false }));
|
|
self.pump();
|
|
}
|
|
|
|
/// More text for an open request (an LLM answer that is still streaming).
|
|
pub fn continue_text(&mut self, id: &str, text: &str, more: bool) {
|
|
if self.queue.iter().any(|r| r.id == id && !r.done) {
|
|
self.backend.tts(id, text, more);
|
|
}
|
|
}
|
|
|
|
/// A sentence of audio arrived from the worker.
|
|
pub fn audio(&mut self, id: &str, seq: u32, pcm: Vec<u8>) {
|
|
if let Some(r) = self.queue.iter_mut().find(|r| r.id == id) {
|
|
r.chunks.insert(seq, pcm);
|
|
self.pump();
|
|
}
|
|
}
|
|
|
|
/// The worker finished a request (all sentences sent, cancelled, or failed).
|
|
pub fn done(&mut self, d: &TtsDone) {
|
|
if let Some(r) = self.queue.iter_mut().find(|r| r.id == d.id) {
|
|
r.done = true;
|
|
r.failed = d.failed;
|
|
self.pump();
|
|
}
|
|
}
|
|
|
|
/// The sink finished playing the current clip.
|
|
pub fn clip_ended(&mut self) {
|
|
self.playing = false;
|
|
self.pump();
|
|
}
|
|
|
|
/// Silence now: stop the clip, drop every queued request, cancel the worker's work.
|
|
pub fn stop(&mut self) -> usize {
|
|
let stopped: Vec<Request> = self.queue.drain(..).collect();
|
|
let was_playing = self.playing;
|
|
self.playing = false;
|
|
for r in stopped.iter().filter(|r| !r.done) {
|
|
self.backend.tts_cancel(&r.id);
|
|
}
|
|
if was_playing {
|
|
self.sink.stop();
|
|
}
|
|
for r in &stopped {
|
|
self.events.push(PlaybackEvent::End(PlaybackEnd { id: r.id.clone(), played: r.played, cancelled: true,
|
|
failed: false }));
|
|
}
|
|
crate::debug_log!("play", "stop requests={} was_playing={was_playing}", stopped.len());
|
|
stopped.len()
|
|
}
|
|
|
|
fn pump(&mut self) {
|
|
while !self.playing {
|
|
let Some(head) = self.queue.front_mut() else { return };
|
|
if !head.chunks.contains_key(&head.next) && head.done {
|
|
// A sentence failed: skip its gap and play what did arrive.
|
|
if let Some(&first) = head.chunks.keys().next() {
|
|
head.next = first;
|
|
}
|
|
}
|
|
if let Some(pcm) = head.chunks.remove(&head.next) {
|
|
head.next += 1;
|
|
head.played += 1;
|
|
if !head.started {
|
|
head.started = true;
|
|
self.events.push(PlaybackEvent::Start(head.id.clone()));
|
|
}
|
|
self.playing = true;
|
|
self.sink.play(pcm);
|
|
return;
|
|
}
|
|
if !head.done {
|
|
return;
|
|
}
|
|
let r = self.queue.pop_front().expect("head exists");
|
|
crate::debug_log!("play", "said id={} sentences={} failed={}", r.id, r.played, r.failed);
|
|
self.events.push(PlaybackEvent::End(PlaybackEnd { id: r.id, played: r.played, cancelled: false,
|
|
failed: r.failed }));
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::{cell::RefCell, rc::Rc};
|
|
|
|
#[derive(Default)]
|
|
struct Log {
|
|
sent: Vec<(String, String, bool)>,
|
|
cancelled: Vec<String>,
|
|
played: Vec<String>,
|
|
stops: usize,
|
|
}
|
|
#[derive(Clone, Default)]
|
|
struct Fake(Rc<RefCell<Log>>);
|
|
impl TtsBackend for Fake {
|
|
fn tts(&self, id: &str, text: &str, more: bool) -> bool {
|
|
self.0.borrow_mut().sent.push((id.into(), text.into(), more));
|
|
true
|
|
}
|
|
fn tts_cancel(&self, id: &str) {
|
|
self.0.borrow_mut().cancelled.push(id.into());
|
|
}
|
|
}
|
|
impl ClipSink for Fake {
|
|
fn play(&mut self, pcm: Vec<u8>) {
|
|
self.0.borrow_mut().played.push(String::from_utf8(pcm).unwrap());
|
|
}
|
|
fn stop(&mut self) {
|
|
self.0.borrow_mut().stops += 1;
|
|
}
|
|
}
|
|
|
|
fn setup() -> (Playback<Fake, Fake>, Fake) {
|
|
let f = Fake::default();
|
|
(Playback::new(f.clone(), f.clone()), f)
|
|
}
|
|
fn done(id: &str, failed: bool) -> TtsDone {
|
|
TtsDone { id: id.into(), chunks: 0, cancelled: false, failed }
|
|
}
|
|
fn end(id: &str, played: u32, cancelled: bool, failed: bool) -> PlaybackEvent {
|
|
PlaybackEvent::End(PlaybackEnd { id: id.into(), played, cancelled, failed })
|
|
}
|
|
fn b(s: &str) -> Vec<u8> {
|
|
s.as_bytes().to_vec()
|
|
}
|
|
|
|
#[test]
|
|
fn plays_the_first_sentence_on_arrival_and_the_next_when_the_clip_ends() {
|
|
let (mut p, f) = setup();
|
|
let id = p.say("하나. 둘.", false);
|
|
assert_eq!(f.0.borrow().sent, vec![(id.clone(), "하나. 둘.".into(), false)]);
|
|
p.audio(&id, 0, b("s0"));
|
|
assert_eq!(f.0.borrow().played, vec!["s0"]);
|
|
p.audio(&id, 1, b("s1"));
|
|
assert_eq!(f.0.borrow().played, vec!["s0"]);
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["s0", "s1"]);
|
|
p.done(&done(&id, false));
|
|
assert_eq!(p.take_events(), vec![PlaybackEvent::Start(id.clone())]);
|
|
p.clip_ended();
|
|
assert_eq!(p.take_events(), vec![end(&id, 2, false, false)]);
|
|
assert!(!p.busy());
|
|
}
|
|
|
|
#[test]
|
|
fn requests_play_in_the_order_they_were_made() {
|
|
let (mut p, f) = setup();
|
|
let a = p.say("a", false);
|
|
let c = p.say("b", false);
|
|
p.audio(&c, 0, b("b0"));
|
|
assert!(f.0.borrow().played.is_empty());
|
|
p.audio(&a, 0, b("a0"));
|
|
p.done(&done(&a, false));
|
|
p.done(&done(&c, false));
|
|
p.clip_ended();
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["a0", "b0"]);
|
|
let ends: Vec<_> = p.take_events().into_iter().filter_map(|e| match e { PlaybackEvent::End(e) => Some(e.id), _ => None }).collect();
|
|
assert_eq!(ends, vec![a, c]);
|
|
}
|
|
|
|
#[test]
|
|
fn sentences_stay_in_order_when_they_arrive_out_of_order() {
|
|
let (mut p, f) = setup();
|
|
let id = p.say("x", true);
|
|
p.audio(&id, 1, b("s1"));
|
|
assert!(f.0.borrow().played.is_empty());
|
|
p.audio(&id, 0, b("s0"));
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["s0", "s1"]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_failed_sentence_is_skipped_and_the_rest_still_play() {
|
|
let (mut p, f) = setup();
|
|
let id = p.say("a. b. c.", false);
|
|
p.audio(&id, 0, b("s0"));
|
|
p.audio(&id, 2, b("s2"));
|
|
p.done(&done(&id, true));
|
|
p.clip_ended();
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["s0", "s2"]);
|
|
assert_eq!(p.take_events().last(), Some(&end(&id, 2, false, true)));
|
|
}
|
|
|
|
#[test]
|
|
fn stop_silences_cancels_unfinished_and_drops_the_queue() {
|
|
let (mut p, f) = setup();
|
|
let a = p.say("a", true);
|
|
let c = p.say("b", false);
|
|
p.audio(&a, 0, b("a0"));
|
|
p.done(&done(&c, false));
|
|
p.take_events();
|
|
assert_eq!(p.stop(), 2);
|
|
assert_eq!(f.0.borrow().stops, 1);
|
|
assert_eq!(f.0.borrow().cancelled, vec![a.clone()]);
|
|
assert_eq!(p.take_events(), vec![end(&a, 1, true, false), end(&c, 0, true, false)]);
|
|
p.audio(&a, 1, b("a1"));
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["a0"]);
|
|
assert!(!p.busy());
|
|
}
|
|
|
|
#[test]
|
|
fn stop_with_nothing_to_say_touches_nothing() {
|
|
let (mut p, f) = setup();
|
|
assert_eq!(p.stop(), 0);
|
|
assert_eq!(f.0.borrow().stops, 0);
|
|
}
|
|
|
|
#[test]
|
|
fn text_can_be_added_to_an_open_request_but_not_a_finished_one() {
|
|
let (mut p, f) = setup();
|
|
let id = p.say("첫 문장.", true);
|
|
p.continue_text(&id, " 둘째 문장.", false);
|
|
p.done(&done(&id, false));
|
|
p.continue_text(&id, "늦은 글", false);
|
|
let sent: Vec<_> = f.0.borrow().sent.iter().map(|(_, t, m)| (t.clone(), *m)).collect();
|
|
assert_eq!(sent, vec![("첫 문장.".into(), true), (" 둘째 문장.".into(), false)]);
|
|
}
|
|
|
|
#[test]
|
|
fn an_adopted_answer_plays_the_brains_audio_without_sending_text() {
|
|
let (mut p, f) = setup();
|
|
let id = p.adopt();
|
|
assert!(f.0.borrow().sent.is_empty(), "the brain speaks it under this id");
|
|
p.audio(&id, 0, b("a0"));
|
|
p.done(&done(&id, false));
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["a0"]);
|
|
assert_eq!(p.take_events(), vec![PlaybackEvent::Start(id.clone()), end(&id, 1, false, false)]);
|
|
assert_ne!(p.adopt(), id, "ids never repeat");
|
|
}
|
|
|
|
#[test]
|
|
fn an_answer_with_no_text_fails_and_the_next_request_plays() {
|
|
let (mut p, f) = setup();
|
|
let id = p.adopt();
|
|
let c = p.say("b", false);
|
|
p.audio(&c, 0, b("b0"));
|
|
assert!(f.0.borrow().played.is_empty());
|
|
p.fail(&id);
|
|
assert_eq!(f.0.borrow().played, vec!["b0"]);
|
|
assert_eq!(p.take_events()[0], end(&id, 0, false, true));
|
|
}
|
|
|
|
#[test]
|
|
fn cancel_stops_one_request_and_leaves_the_others() {
|
|
let (mut p, f) = setup();
|
|
let a = p.say("a", true);
|
|
let c = p.say("b", false);
|
|
p.audio(&a, 0, b("a0"));
|
|
p.audio(&c, 0, b("b0"));
|
|
p.take_events();
|
|
p.cancel(&a);
|
|
assert_eq!(f.0.borrow().stops, 1);
|
|
assert_eq!(f.0.borrow().cancelled, vec![a.clone()]);
|
|
assert_eq!(f.0.borrow().played, vec!["a0", "b0"], "the next request plays at once");
|
|
assert_eq!(p.take_events(), vec![end(&a, 1, true, false), PlaybackEvent::Start(c.clone())]);
|
|
// A queued request that has not started is cancelled without touching the clip.
|
|
let d = p.say("d", false);
|
|
p.cancel(&d);
|
|
assert_eq!(f.0.borrow().stops, 1);
|
|
assert_eq!(p.take_events(), vec![end(&d, 0, true, false)]);
|
|
p.cancel("nope");
|
|
assert!(p.take_events().is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn a_request_the_worker_never_accepted_fails_without_blocking_the_next() {
|
|
let (mut p, f) = setup();
|
|
let a = p.say("a", false);
|
|
let c = p.say("b", false);
|
|
p.done(&done(&a, true));
|
|
p.audio(&c, 0, b("b0"));
|
|
assert_eq!(f.0.borrow().played, vec!["b0"]);
|
|
assert_eq!(p.take_events()[0], end(&a, 0, false, true));
|
|
}
|
|
|
|
#[test]
|
|
fn a_clip_plays_at_once_then_the_answer_queued_after_it() {
|
|
let (mut p, f) = setup();
|
|
let cue = p.clip(b("ding"));
|
|
let id = p.adopt();
|
|
assert_eq!(f.0.borrow().played, vec!["ding"], "the clip needs no worker");
|
|
assert!(f.0.borrow().sent.is_empty());
|
|
p.audio(&id, 0, b("a0"));
|
|
assert_eq!(f.0.borrow().played, vec!["ding"]);
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["ding", "a0"]);
|
|
assert_eq!(p.take_events(), vec![PlaybackEvent::Start(cue.clone()), end(&cue, 1, false, false),
|
|
PlaybackEvent::Start(id.clone())]);
|
|
// A stop drops a clip still waiting its turn, without asking the worker to cancel it.
|
|
let cue2 = p.clip(b("ding"));
|
|
p.stop();
|
|
assert_eq!(f.0.borrow().cancelled, vec![id.clone()]);
|
|
assert!(p.take_events().contains(&end(&cue2, 0, true, false)));
|
|
}
|
|
}
|