- Playback sends how long the clip playing when a request was stopped had played (tts_played last_ms); the worker cuts the last sentence of that recording to it. - botBrowser open takes the search words as plain `query`: the tool puts them, encoded, into the address's empty search parameter or types them into the site's search box. The description tells the model to search this way and never put words into an address itself (live: 아이유 hand-encoded into 아이우). Eval (qwen3:8b, 3 runs): right words reach YouTube 1/3 -> 3/3. - When a page's main video does not play (also after the tool pressed play, or it failed to load), a note after the page says it is not playing and not to say it plays. Eval: a paused video reported as playing 1/3 -> 0/3. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
484 lines
18 KiB
Rust
484 lines
18 KiB
Rust
//! The bot's mouth: turns text into queued speech, one request after another.
|
|
//!
|
|
//! Each request goes to the voice worker as TTS; the worker sends back one PCM chunk per sentence.
|
|
//! Chunks play in order as soon as they arrive, so the first sentence plays while later ones are still
|
|
//! being made. Requests play first come, first served. `stop()` silences everything at once and
|
|
//! cancels what the worker has not made yet.
|
|
|
|
use std::collections::{BTreeMap, VecDeque};
|
|
|
|
use crate::worker_client::TtsDone;
|
|
|
|
pub trait TtsBackend {
|
|
/// Send text for request `id`; false when the worker is not connected (the request then fails).
|
|
fn tts(&self, id: &str, text: &str, more: bool) -> bool;
|
|
fn tts_cancel(&self, id: &str);
|
|
/// Request `id` ended after `played` of its sentences started playing, the last of them for
|
|
/// `last_ms` before it was stopped (None: played to its end). The worker's recording keeps that
|
|
/// much (voice_worker.spec.md, "Recordings").
|
|
fn tts_played(&self, id: &str, played: u32, cancelled: bool, last_ms: Option<u32>);
|
|
}
|
|
|
|
/// Plays one PCM clip at a time; its owner calls `Playback::clip_ended()` when the clip finishes.
|
|
pub trait ClipSink {
|
|
fn play(&mut self, pcm: Vec<u8>);
|
|
fn stop(&mut self);
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct PlaybackEnd {
|
|
pub id: String,
|
|
/// Sentences that were played.
|
|
pub played: u32,
|
|
pub cancelled: bool,
|
|
pub failed: bool,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub enum PlaybackEvent {
|
|
Start(String),
|
|
End(PlaybackEnd),
|
|
}
|
|
|
|
struct Request {
|
|
id: String,
|
|
chunks: BTreeMap<u32, Vec<u8>>,
|
|
next: u32,
|
|
done: bool,
|
|
failed: bool,
|
|
played: u32,
|
|
started: bool,
|
|
}
|
|
|
|
pub struct Playback<B: TtsBackend, S: ClipSink> {
|
|
backend: B,
|
|
sink: S,
|
|
queue: VecDeque<Request>,
|
|
playing: bool,
|
|
counter: u64,
|
|
events: Vec<PlaybackEvent>,
|
|
/// When the clip now playing started.
|
|
clip_started: Option<std::time::Instant>,
|
|
}
|
|
|
|
impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
|
pub fn new(backend: B, sink: S) -> Self {
|
|
Self { backend, sink, queue: VecDeque::new(), playing: false, counter: 0, events: vec![], clip_started: None }
|
|
}
|
|
|
|
/// True while a clip is playing or a request is waiting for its audio.
|
|
pub fn busy(&self) -> bool {
|
|
self.playing || !self.queue.is_empty()
|
|
}
|
|
|
|
/// Start and end events since the last call.
|
|
pub fn take_events(&mut self) -> Vec<PlaybackEvent> {
|
|
std::mem::take(&mut self.events)
|
|
}
|
|
|
|
/// Queue `text` as a new request; with `more`, add text later with `continue_text`. Returns its id.
|
|
pub fn say(&mut self, text: &str, more: bool) -> String {
|
|
self.counter += 1;
|
|
let id = format!("r{}", self.counter);
|
|
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false,
|
|
played: 0, started: false });
|
|
crate::debug_log!("play", "say id={id} chars={} more={more} queued={}", text.chars().count(), self.queue.len());
|
|
self.backend.tts(&id, text, more);
|
|
id
|
|
}
|
|
|
|
/// Queue a request whose speech the worker makes from the brain's answer (`reply` with `speak`),
|
|
/// under the returned id. Nothing is sent here.
|
|
pub fn adopt(&mut self) -> String {
|
|
self.counter += 1;
|
|
let id = format!("r{}", self.counter);
|
|
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false,
|
|
played: 0, started: false });
|
|
crate::debug_log!("play", "answer id={id} queued={}", self.queue.len());
|
|
id
|
|
}
|
|
|
|
/// Queue a clip the app already has (the earcon) as a request of its own. Returns its id.
|
|
pub fn clip(&mut self, pcm: Vec<u8>) -> String {
|
|
self.counter += 1;
|
|
let id = format!("r{}", self.counter);
|
|
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::from([(0, pcm)]), next: 0, done: true,
|
|
failed: false, played: 0, started: false });
|
|
crate::debug_log!("play", "clip id={id} queued={}", self.queue.len());
|
|
self.pump();
|
|
id
|
|
}
|
|
|
|
/// End a request that will get no audio (the brain gave no answer).
|
|
pub fn fail(&mut self, id: &str) {
|
|
if let Some(r) = self.queue.iter_mut().find(|r| r.id == id && !r.done) {
|
|
r.done = true;
|
|
r.failed = true;
|
|
self.pump();
|
|
}
|
|
}
|
|
|
|
/// Stop one request: its clip if it is playing, its unsent sentences at the worker. The rest of the
|
|
/// queue goes on.
|
|
pub fn cancel(&mut self, id: &str) {
|
|
let Some(pos) = self.queue.iter().position(|r| r.id == id) else { return };
|
|
let r = self.queue.remove(pos).expect("found");
|
|
if !r.done {
|
|
self.backend.tts_cancel(&r.id);
|
|
}
|
|
let mut last_ms = None;
|
|
if pos == 0 && self.playing {
|
|
self.playing = false;
|
|
last_ms = self.clip_elapsed_ms();
|
|
self.sink.stop();
|
|
}
|
|
crate::debug_log!("play", "cancel id={} played={}", r.id, r.played);
|
|
self.ended(PlaybackEnd { id: r.id, played: r.played, cancelled: true, failed: false }, last_ms);
|
|
self.pump();
|
|
}
|
|
|
|
/// More text for an open request (an LLM answer that is still streaming).
|
|
pub fn continue_text(&mut self, id: &str, text: &str, more: bool) {
|
|
if self.queue.iter().any(|r| r.id == id && !r.done) {
|
|
self.backend.tts(id, text, more);
|
|
}
|
|
}
|
|
|
|
/// A sentence of audio arrived from the worker.
|
|
pub fn audio(&mut self, id: &str, seq: u32, pcm: Vec<u8>) {
|
|
if let Some(r) = self.queue.iter_mut().find(|r| r.id == id) {
|
|
r.chunks.insert(seq, pcm);
|
|
self.pump();
|
|
}
|
|
}
|
|
|
|
/// The worker finished a request (all sentences sent, cancelled, or failed).
|
|
pub fn done(&mut self, d: &TtsDone) {
|
|
if let Some(r) = self.queue.iter_mut().find(|r| r.id == d.id) {
|
|
r.done = true;
|
|
r.failed = d.failed;
|
|
self.pump();
|
|
}
|
|
}
|
|
|
|
/// The sink finished playing the current clip.
|
|
pub fn clip_ended(&mut self) {
|
|
self.playing = false;
|
|
self.pump();
|
|
}
|
|
|
|
/// Silence now: stop the clip, drop every queued request, cancel the worker's work.
|
|
pub fn stop(&mut self) -> usize {
|
|
let stopped: Vec<Request> = self.queue.drain(..).collect();
|
|
let was_playing = self.playing;
|
|
let head_ms = if was_playing { self.clip_elapsed_ms() } else { None };
|
|
self.playing = false;
|
|
for r in stopped.iter().filter(|r| !r.done) {
|
|
self.backend.tts_cancel(&r.id);
|
|
}
|
|
if was_playing {
|
|
self.sink.stop();
|
|
}
|
|
for (i, r) in stopped.iter().enumerate() {
|
|
let last_ms = if i == 0 { head_ms } else { None };
|
|
self.ended(PlaybackEnd { id: r.id.clone(), played: r.played, cancelled: true, failed: false }, last_ms);
|
|
}
|
|
crate::debug_log!("play", "stop requests={} was_playing={was_playing}", stopped.len());
|
|
stopped.len()
|
|
}
|
|
|
|
/// A request ended: tell the worker how much of it was played, then the owner.
|
|
fn ended(&mut self, end: PlaybackEnd, last_ms: Option<u32>) {
|
|
self.backend.tts_played(&end.id, end.played, end.cancelled, last_ms);
|
|
self.events.push(PlaybackEvent::End(end));
|
|
}
|
|
|
|
/// How long the clip now playing has played.
|
|
fn clip_elapsed_ms(&self) -> Option<u32> {
|
|
self.clip_started.map(|t| t.elapsed().as_millis().min(u32::MAX as u128) as u32)
|
|
}
|
|
|
|
fn pump(&mut self) {
|
|
while !self.playing {
|
|
let Some(head) = self.queue.front_mut() else { return };
|
|
if !head.chunks.contains_key(&head.next) && head.done {
|
|
// A sentence failed: skip its gap and play what did arrive.
|
|
if let Some(&first) = head.chunks.keys().next() {
|
|
head.next = first;
|
|
}
|
|
}
|
|
if let Some(pcm) = head.chunks.remove(&head.next) {
|
|
head.next += 1;
|
|
head.played += 1;
|
|
if !head.started {
|
|
head.started = true;
|
|
self.events.push(PlaybackEvent::Start(head.id.clone()));
|
|
}
|
|
self.playing = true;
|
|
self.clip_started = Some(std::time::Instant::now());
|
|
self.sink.play(pcm);
|
|
return;
|
|
}
|
|
if !head.done {
|
|
return;
|
|
}
|
|
let r = self.queue.pop_front().expect("head exists");
|
|
crate::debug_log!("play", "said id={} sentences={} failed={}", r.id, r.played, r.failed);
|
|
self.ended(PlaybackEnd { id: r.id, played: r.played, cancelled: false, failed: r.failed }, None);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::{cell::RefCell, rc::Rc};
|
|
|
|
#[derive(Default)]
|
|
struct Log {
|
|
sent: Vec<(String, String, bool)>,
|
|
cancelled: Vec<String>,
|
|
played: Vec<String>,
|
|
stops: usize,
|
|
reported: Vec<(String, u32, bool, Option<u32>)>,
|
|
}
|
|
#[derive(Clone, Default)]
|
|
struct Fake(Rc<RefCell<Log>>);
|
|
impl TtsBackend for Fake {
|
|
fn tts(&self, id: &str, text: &str, more: bool) -> bool {
|
|
self.0.borrow_mut().sent.push((id.into(), text.into(), more));
|
|
true
|
|
}
|
|
fn tts_cancel(&self, id: &str) {
|
|
self.0.borrow_mut().cancelled.push(id.into());
|
|
}
|
|
fn tts_played(&self, id: &str, played: u32, cancelled: bool, last_ms: Option<u32>) {
|
|
self.0.borrow_mut().reported.push((id.into(), played, cancelled, last_ms));
|
|
}
|
|
}
|
|
impl ClipSink for Fake {
|
|
fn play(&mut self, pcm: Vec<u8>) {
|
|
self.0.borrow_mut().played.push(String::from_utf8(pcm).unwrap());
|
|
}
|
|
fn stop(&mut self) {
|
|
self.0.borrow_mut().stops += 1;
|
|
}
|
|
}
|
|
|
|
fn setup() -> (Playback<Fake, Fake>, Fake) {
|
|
let f = Fake::default();
|
|
(Playback::new(f.clone(), f.clone()), f)
|
|
}
|
|
fn done(id: &str, failed: bool) -> TtsDone {
|
|
TtsDone { id: id.into(), chunks: 0, cancelled: false, failed }
|
|
}
|
|
fn end(id: &str, played: u32, cancelled: bool, failed: bool) -> PlaybackEvent {
|
|
PlaybackEvent::End(PlaybackEnd { id: id.into(), played, cancelled, failed })
|
|
}
|
|
fn b(s: &str) -> Vec<u8> {
|
|
s.as_bytes().to_vec()
|
|
}
|
|
|
|
#[test]
|
|
fn plays_the_first_sentence_on_arrival_and_the_next_when_the_clip_ends() {
|
|
let (mut p, f) = setup();
|
|
let id = p.say("하나. 둘.", false);
|
|
assert_eq!(f.0.borrow().sent, vec![(id.clone(), "하나. 둘.".into(), false)]);
|
|
p.audio(&id, 0, b("s0"));
|
|
assert_eq!(f.0.borrow().played, vec!["s0"]);
|
|
p.audio(&id, 1, b("s1"));
|
|
assert_eq!(f.0.borrow().played, vec!["s0"]);
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["s0", "s1"]);
|
|
p.done(&done(&id, false));
|
|
assert_eq!(p.take_events(), vec![PlaybackEvent::Start(id.clone())]);
|
|
p.clip_ended();
|
|
assert_eq!(p.take_events(), vec![end(&id, 2, false, false)]);
|
|
assert!(!p.busy());
|
|
}
|
|
|
|
#[test]
|
|
fn the_worker_is_told_how_much_of_each_request_was_played() {
|
|
let (mut p, f) = setup();
|
|
let a = p.say("하나. 둘.", false);
|
|
p.audio(&a, 0, b("a0"));
|
|
p.audio(&a, 1, b("a1"));
|
|
p.done(&done(&a, false));
|
|
p.clip_ended();
|
|
p.clip_ended();
|
|
let c = p.say("셋. 넷.", false);
|
|
p.audio(&c, 0, b("c0"));
|
|
p.audio(&c, 1, b("c1"));
|
|
p.done(&done(&c, false));
|
|
std::thread::sleep(std::time::Duration::from_millis(60));
|
|
p.cancel(&c); // stopped about 60 ms into its first sentence
|
|
let e = p.say("여섯.", false);
|
|
p.audio(&e, 0, b("e0"));
|
|
let d = p.say("다섯.", false);
|
|
p.stop(); // e stopped at once, d before any sound
|
|
let r = f.0.borrow().reported.clone();
|
|
assert_eq!(r[0], (a, 2, false, None), "played to its end");
|
|
let (id, n, cancelled, ms) = r[1].clone();
|
|
assert_eq!((id, n, cancelled), (c, 1, true));
|
|
assert!(ms.is_some_and(|ms| (50..1000).contains(&ms)), "how long the stopped sentence played: {ms:?}");
|
|
assert_eq!((r[2].0.clone(), r[2].1, r[2].2), (e, 1, true));
|
|
assert!(r[2].3.is_some_and(|ms| ms < 50));
|
|
assert_eq!(r[3], (d, 0, true, None));
|
|
}
|
|
|
|
#[test]
|
|
fn requests_play_in_the_order_they_were_made() {
|
|
let (mut p, f) = setup();
|
|
let a = p.say("a", false);
|
|
let c = p.say("b", false);
|
|
p.audio(&c, 0, b("b0"));
|
|
assert!(f.0.borrow().played.is_empty());
|
|
p.audio(&a, 0, b("a0"));
|
|
p.done(&done(&a, false));
|
|
p.done(&done(&c, false));
|
|
p.clip_ended();
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["a0", "b0"]);
|
|
let ends: Vec<_> = p.take_events().into_iter().filter_map(|e| match e { PlaybackEvent::End(e) => Some(e.id), _ => None }).collect();
|
|
assert_eq!(ends, vec![a, c]);
|
|
}
|
|
|
|
#[test]
|
|
fn sentences_stay_in_order_when_they_arrive_out_of_order() {
|
|
let (mut p, f) = setup();
|
|
let id = p.say("x", true);
|
|
p.audio(&id, 1, b("s1"));
|
|
assert!(f.0.borrow().played.is_empty());
|
|
p.audio(&id, 0, b("s0"));
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["s0", "s1"]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_failed_sentence_is_skipped_and_the_rest_still_play() {
|
|
let (mut p, f) = setup();
|
|
let id = p.say("a. b. c.", false);
|
|
p.audio(&id, 0, b("s0"));
|
|
p.audio(&id, 2, b("s2"));
|
|
p.done(&done(&id, true));
|
|
p.clip_ended();
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["s0", "s2"]);
|
|
assert_eq!(p.take_events().last(), Some(&end(&id, 2, false, true)));
|
|
}
|
|
|
|
#[test]
|
|
fn stop_silences_cancels_unfinished_and_drops_the_queue() {
|
|
let (mut p, f) = setup();
|
|
let a = p.say("a", true);
|
|
let c = p.say("b", false);
|
|
p.audio(&a, 0, b("a0"));
|
|
p.done(&done(&c, false));
|
|
p.take_events();
|
|
assert_eq!(p.stop(), 2);
|
|
assert_eq!(f.0.borrow().stops, 1);
|
|
assert_eq!(f.0.borrow().cancelled, vec![a.clone()]);
|
|
assert_eq!(p.take_events(), vec![end(&a, 1, true, false), end(&c, 0, true, false)]);
|
|
p.audio(&a, 1, b("a1"));
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["a0"]);
|
|
assert!(!p.busy());
|
|
}
|
|
|
|
#[test]
|
|
fn stop_with_nothing_to_say_touches_nothing() {
|
|
let (mut p, f) = setup();
|
|
assert_eq!(p.stop(), 0);
|
|
assert_eq!(f.0.borrow().stops, 0);
|
|
}
|
|
|
|
#[test]
|
|
fn text_can_be_added_to_an_open_request_but_not_a_finished_one() {
|
|
let (mut p, f) = setup();
|
|
let id = p.say("첫 문장.", true);
|
|
p.continue_text(&id, " 둘째 문장.", false);
|
|
p.done(&done(&id, false));
|
|
p.continue_text(&id, "늦은 글", false);
|
|
let sent: Vec<_> = f.0.borrow().sent.iter().map(|(_, t, m)| (t.clone(), *m)).collect();
|
|
assert_eq!(sent, vec![("첫 문장.".into(), true), (" 둘째 문장.".into(), false)]);
|
|
}
|
|
|
|
#[test]
|
|
fn an_adopted_answer_plays_the_brains_audio_without_sending_text() {
|
|
let (mut p, f) = setup();
|
|
let id = p.adopt();
|
|
assert!(f.0.borrow().sent.is_empty(), "the brain speaks it under this id");
|
|
p.audio(&id, 0, b("a0"));
|
|
p.done(&done(&id, false));
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["a0"]);
|
|
assert_eq!(p.take_events(), vec![PlaybackEvent::Start(id.clone()), end(&id, 1, false, false)]);
|
|
assert_ne!(p.adopt(), id, "ids never repeat");
|
|
}
|
|
|
|
#[test]
|
|
fn an_answer_with_no_text_fails_and_the_next_request_plays() {
|
|
let (mut p, f) = setup();
|
|
let id = p.adopt();
|
|
let c = p.say("b", false);
|
|
p.audio(&c, 0, b("b0"));
|
|
assert!(f.0.borrow().played.is_empty());
|
|
p.fail(&id);
|
|
assert_eq!(f.0.borrow().played, vec!["b0"]);
|
|
assert_eq!(p.take_events()[0], end(&id, 0, false, true));
|
|
}
|
|
|
|
#[test]
|
|
fn cancel_stops_one_request_and_leaves_the_others() {
|
|
let (mut p, f) = setup();
|
|
let a = p.say("a", true);
|
|
let c = p.say("b", false);
|
|
p.audio(&a, 0, b("a0"));
|
|
p.audio(&c, 0, b("b0"));
|
|
p.take_events();
|
|
p.cancel(&a);
|
|
assert_eq!(f.0.borrow().stops, 1);
|
|
assert_eq!(f.0.borrow().cancelled, vec![a.clone()]);
|
|
assert_eq!(f.0.borrow().played, vec!["a0", "b0"], "the next request plays at once");
|
|
assert_eq!(p.take_events(), vec![end(&a, 1, true, false), PlaybackEvent::Start(c.clone())]);
|
|
// A queued request that has not started is cancelled without touching the clip.
|
|
let d = p.say("d", false);
|
|
p.cancel(&d);
|
|
assert_eq!(f.0.borrow().stops, 1);
|
|
assert_eq!(p.take_events(), vec![end(&d, 0, true, false)]);
|
|
p.cancel("nope");
|
|
assert!(p.take_events().is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn a_request_the_worker_never_accepted_fails_without_blocking_the_next() {
|
|
let (mut p, f) = setup();
|
|
let a = p.say("a", false);
|
|
let c = p.say("b", false);
|
|
p.done(&done(&a, true));
|
|
p.audio(&c, 0, b("b0"));
|
|
assert_eq!(f.0.borrow().played, vec!["b0"]);
|
|
assert_eq!(p.take_events()[0], end(&a, 0, false, true));
|
|
}
|
|
|
|
#[test]
|
|
fn a_clip_plays_at_once_then_the_answer_queued_after_it() {
|
|
let (mut p, f) = setup();
|
|
let cue = p.clip(b("ding"));
|
|
let id = p.adopt();
|
|
assert_eq!(f.0.borrow().played, vec!["ding"], "the clip needs no worker");
|
|
assert!(f.0.borrow().sent.is_empty());
|
|
p.audio(&id, 0, b("a0"));
|
|
assert_eq!(f.0.borrow().played, vec!["ding"]);
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["ding", "a0"]);
|
|
assert_eq!(p.take_events(), vec![PlaybackEvent::Start(cue.clone()), end(&cue, 1, false, false),
|
|
PlaybackEvent::Start(id.clone())]);
|
|
// A stop drops a clip still waiting its turn, without asking the worker to cancel it.
|
|
let cue2 = p.clip(b("ding"));
|
|
p.stop();
|
|
assert_eq!(f.0.borrow().cancelled, vec![id.clone()]);
|
|
assert!(p.take_events().contains(&end(&cue2, 0, true, false)));
|
|
}
|
|
}
|