The user asked for a faster, real app instead of Electron. The app is now a Rust crate (serenity + songbird); the GUI in M5 will be a Tauri 2 shell on the system WebView2. - Same behaviour as the Electron core: one voice channel, per-speaker PCM to the voice worker, speaking_end and speaker_leave, sentence playback with stop and cancel, transcription notice on join, slash command join/leave/stop (create-or-update), registered chat channels with live events, history and sends that never ping, worker process with a fresh token and restart backoff. - songbird 0.6.1 garbles about a third of received frames under DAVE (its decoder keeps the bytes DAVE decryption leaves at the end of the payload; a 20 ms stepped test tone showed repeats, skips and reordering, while sending was exact). songbird now only decrypts; the core reorders packets in a small window (Opus FEC for the last lost frame) and decodes them with libopus itself. - Live test on real Discord with two bots: five sentences heard (CER 0/0/0/5/12.5%), 0.71-0.90 s from end of speech to final, speech read back at 3.4%, sound gone 0.08 s after stop. The release binary uses about 16 MB of memory; a Windows build (MinGW) logs in under Wine. - Spec moved to app/src/discord_core.spec.md; architecture, decisions, roadmap and pc-features updated. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
316 lines
11 KiB
Rust
316 lines
11 KiB
Rust
//! The bot's mouth: turns text into queued speech, one request after another.
|
|
//!
|
|
//! Each request goes to the voice worker as TTS; the worker sends back one PCM chunk per sentence.
|
|
//! Chunks play in order as soon as they arrive, so the first sentence plays while later ones are still
|
|
//! being made. Requests play first come, first served. `stop()` silences everything at once and
|
|
//! cancels what the worker has not made yet.
|
|
|
|
use std::collections::{BTreeMap, VecDeque};
|
|
|
|
use crate::worker_client::TtsDone;
|
|
|
|
pub trait TtsBackend {
|
|
/// Send text for request `id`; false when the worker is not connected (the request then fails).
|
|
fn tts(&self, id: &str, text: &str, more: bool) -> bool;
|
|
fn tts_cancel(&self, id: &str);
|
|
}
|
|
|
|
/// Plays one PCM clip at a time; its owner calls `Playback::clip_ended()` when the clip finishes.
|
|
pub trait ClipSink {
|
|
fn play(&mut self, pcm: Vec<u8>);
|
|
fn stop(&mut self);
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct PlaybackEnd {
|
|
pub id: String,
|
|
/// Sentences that were played.
|
|
pub played: u32,
|
|
pub cancelled: bool,
|
|
pub failed: bool,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub enum PlaybackEvent {
|
|
Start(String),
|
|
End(PlaybackEnd),
|
|
}
|
|
|
|
struct Request {
|
|
id: String,
|
|
chunks: BTreeMap<u32, Vec<u8>>,
|
|
next: u32,
|
|
done: bool,
|
|
failed: bool,
|
|
played: u32,
|
|
started: bool,
|
|
}
|
|
|
|
pub struct Playback<B: TtsBackend, S: ClipSink> {
|
|
backend: B,
|
|
sink: S,
|
|
queue: VecDeque<Request>,
|
|
playing: bool,
|
|
counter: u64,
|
|
events: Vec<PlaybackEvent>,
|
|
}
|
|
|
|
impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
|
pub fn new(backend: B, sink: S) -> Self {
|
|
Self { backend, sink, queue: VecDeque::new(), playing: false, counter: 0, events: vec![] }
|
|
}
|
|
|
|
/// True while a clip is playing or a request is waiting for its audio.
|
|
pub fn busy(&self) -> bool {
|
|
self.playing || !self.queue.is_empty()
|
|
}
|
|
|
|
/// Start and end events since the last call.
|
|
pub fn take_events(&mut self) -> Vec<PlaybackEvent> {
|
|
std::mem::take(&mut self.events)
|
|
}
|
|
|
|
/// Queue `text` as a new request; with `more`, add text later with `continue_text`. Returns its id.
|
|
pub fn say(&mut self, text: &str, more: bool) -> String {
|
|
self.counter += 1;
|
|
let id = format!("r{}", self.counter);
|
|
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false,
|
|
played: 0, started: false });
|
|
crate::debug_log!("play", "say id={id} chars={} more={more} queued={}", text.chars().count(), self.queue.len());
|
|
self.backend.tts(&id, text, more);
|
|
id
|
|
}
|
|
|
|
/// More text for an open request (an LLM answer that is still streaming).
|
|
pub fn continue_text(&mut self, id: &str, text: &str, more: bool) {
|
|
if self.queue.iter().any(|r| r.id == id && !r.done) {
|
|
self.backend.tts(id, text, more);
|
|
}
|
|
}
|
|
|
|
/// A sentence of audio arrived from the worker.
|
|
pub fn audio(&mut self, id: &str, seq: u32, pcm: Vec<u8>) {
|
|
if let Some(r) = self.queue.iter_mut().find(|r| r.id == id) {
|
|
r.chunks.insert(seq, pcm);
|
|
self.pump();
|
|
}
|
|
}
|
|
|
|
/// The worker finished a request (all sentences sent, cancelled, or failed).
|
|
pub fn done(&mut self, d: &TtsDone) {
|
|
if let Some(r) = self.queue.iter_mut().find(|r| r.id == d.id) {
|
|
r.done = true;
|
|
r.failed = d.failed;
|
|
self.pump();
|
|
}
|
|
}
|
|
|
|
/// The sink finished playing the current clip.
|
|
pub fn clip_ended(&mut self) {
|
|
self.playing = false;
|
|
self.pump();
|
|
}
|
|
|
|
/// Silence now: stop the clip, drop every queued request, cancel the worker's work.
|
|
pub fn stop(&mut self) -> usize {
|
|
let stopped: Vec<Request> = self.queue.drain(..).collect();
|
|
let was_playing = self.playing;
|
|
self.playing = false;
|
|
for r in stopped.iter().filter(|r| !r.done) {
|
|
self.backend.tts_cancel(&r.id);
|
|
}
|
|
if was_playing {
|
|
self.sink.stop();
|
|
}
|
|
for r in &stopped {
|
|
self.events.push(PlaybackEvent::End(PlaybackEnd { id: r.id.clone(), played: r.played, cancelled: true,
|
|
failed: false }));
|
|
}
|
|
crate::debug_log!("play", "stop requests={} was_playing={was_playing}", stopped.len());
|
|
stopped.len()
|
|
}
|
|
|
|
fn pump(&mut self) {
|
|
while !self.playing {
|
|
let Some(head) = self.queue.front_mut() else { return };
|
|
if !head.chunks.contains_key(&head.next) && head.done {
|
|
// A sentence failed: skip its gap and play what did arrive.
|
|
if let Some(&first) = head.chunks.keys().next() {
|
|
head.next = first;
|
|
}
|
|
}
|
|
if let Some(pcm) = head.chunks.remove(&head.next) {
|
|
head.next += 1;
|
|
head.played += 1;
|
|
if !head.started {
|
|
head.started = true;
|
|
self.events.push(PlaybackEvent::Start(head.id.clone()));
|
|
}
|
|
self.playing = true;
|
|
self.sink.play(pcm);
|
|
return;
|
|
}
|
|
if !head.done {
|
|
return;
|
|
}
|
|
let r = self.queue.pop_front().expect("head exists");
|
|
crate::debug_log!("play", "said id={} sentences={} failed={}", r.id, r.played, r.failed);
|
|
self.events.push(PlaybackEvent::End(PlaybackEnd { id: r.id, played: r.played, cancelled: false,
|
|
failed: r.failed }));
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::{cell::RefCell, rc::Rc};
|
|
|
|
#[derive(Default)]
|
|
struct Log {
|
|
sent: Vec<(String, String, bool)>,
|
|
cancelled: Vec<String>,
|
|
played: Vec<String>,
|
|
stops: usize,
|
|
}
|
|
#[derive(Clone, Default)]
|
|
struct Fake(Rc<RefCell<Log>>);
|
|
impl TtsBackend for Fake {
|
|
fn tts(&self, id: &str, text: &str, more: bool) -> bool {
|
|
self.0.borrow_mut().sent.push((id.into(), text.into(), more));
|
|
true
|
|
}
|
|
fn tts_cancel(&self, id: &str) {
|
|
self.0.borrow_mut().cancelled.push(id.into());
|
|
}
|
|
}
|
|
impl ClipSink for Fake {
|
|
fn play(&mut self, pcm: Vec<u8>) {
|
|
self.0.borrow_mut().played.push(String::from_utf8(pcm).unwrap());
|
|
}
|
|
fn stop(&mut self) {
|
|
self.0.borrow_mut().stops += 1;
|
|
}
|
|
}
|
|
|
|
fn setup() -> (Playback<Fake, Fake>, Fake) {
|
|
let f = Fake::default();
|
|
(Playback::new(f.clone(), f.clone()), f)
|
|
}
|
|
fn done(id: &str, failed: bool) -> TtsDone {
|
|
TtsDone { id: id.into(), chunks: 0, cancelled: false, failed }
|
|
}
|
|
fn end(id: &str, played: u32, cancelled: bool, failed: bool) -> PlaybackEvent {
|
|
PlaybackEvent::End(PlaybackEnd { id: id.into(), played, cancelled, failed })
|
|
}
|
|
fn b(s: &str) -> Vec<u8> {
|
|
s.as_bytes().to_vec()
|
|
}
|
|
|
|
#[test]
|
|
fn plays_the_first_sentence_on_arrival_and_the_next_when_the_clip_ends() {
|
|
let (mut p, f) = setup();
|
|
let id = p.say("하나. 둘.", false);
|
|
assert_eq!(f.0.borrow().sent, vec![(id.clone(), "하나. 둘.".into(), false)]);
|
|
p.audio(&id, 0, b("s0"));
|
|
assert_eq!(f.0.borrow().played, vec!["s0"]);
|
|
p.audio(&id, 1, b("s1"));
|
|
assert_eq!(f.0.borrow().played, vec!["s0"]);
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["s0", "s1"]);
|
|
p.done(&done(&id, false));
|
|
assert_eq!(p.take_events(), vec![PlaybackEvent::Start(id.clone())]);
|
|
p.clip_ended();
|
|
assert_eq!(p.take_events(), vec![end(&id, 2, false, false)]);
|
|
assert!(!p.busy());
|
|
}
|
|
|
|
#[test]
|
|
fn requests_play_in_the_order_they_were_made() {
|
|
let (mut p, f) = setup();
|
|
let a = p.say("a", false);
|
|
let c = p.say("b", false);
|
|
p.audio(&c, 0, b("b0"));
|
|
assert!(f.0.borrow().played.is_empty());
|
|
p.audio(&a, 0, b("a0"));
|
|
p.done(&done(&a, false));
|
|
p.done(&done(&c, false));
|
|
p.clip_ended();
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["a0", "b0"]);
|
|
let ends: Vec<_> = p.take_events().into_iter().filter_map(|e| match e { PlaybackEvent::End(e) => Some(e.id), _ => None }).collect();
|
|
assert_eq!(ends, vec![a, c]);
|
|
}
|
|
|
|
#[test]
|
|
fn sentences_stay_in_order_when_they_arrive_out_of_order() {
|
|
let (mut p, f) = setup();
|
|
let id = p.say("x", true);
|
|
p.audio(&id, 1, b("s1"));
|
|
assert!(f.0.borrow().played.is_empty());
|
|
p.audio(&id, 0, b("s0"));
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["s0", "s1"]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_failed_sentence_is_skipped_and_the_rest_still_play() {
|
|
let (mut p, f) = setup();
|
|
let id = p.say("a. b. c.", false);
|
|
p.audio(&id, 0, b("s0"));
|
|
p.audio(&id, 2, b("s2"));
|
|
p.done(&done(&id, true));
|
|
p.clip_ended();
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["s0", "s2"]);
|
|
assert_eq!(p.take_events().last(), Some(&end(&id, 2, false, true)));
|
|
}
|
|
|
|
#[test]
|
|
fn stop_silences_cancels_unfinished_and_drops_the_queue() {
|
|
let (mut p, f) = setup();
|
|
let a = p.say("a", true);
|
|
let c = p.say("b", false);
|
|
p.audio(&a, 0, b("a0"));
|
|
p.done(&done(&c, false));
|
|
p.take_events();
|
|
assert_eq!(p.stop(), 2);
|
|
assert_eq!(f.0.borrow().stops, 1);
|
|
assert_eq!(f.0.borrow().cancelled, vec![a.clone()]);
|
|
assert_eq!(p.take_events(), vec![end(&a, 1, true, false), end(&c, 0, true, false)]);
|
|
p.audio(&a, 1, b("a1"));
|
|
p.clip_ended();
|
|
assert_eq!(f.0.borrow().played, vec!["a0"]);
|
|
assert!(!p.busy());
|
|
}
|
|
|
|
#[test]
|
|
fn stop_with_nothing_to_say_touches_nothing() {
|
|
let (mut p, f) = setup();
|
|
assert_eq!(p.stop(), 0);
|
|
assert_eq!(f.0.borrow().stops, 0);
|
|
}
|
|
|
|
#[test]
|
|
fn text_can_be_added_to_an_open_request_but_not_a_finished_one() {
|
|
let (mut p, f) = setup();
|
|
let id = p.say("첫 문장.", true);
|
|
p.continue_text(&id, " 둘째 문장.", false);
|
|
p.done(&done(&id, false));
|
|
p.continue_text(&id, "늦은 글", false);
|
|
let sent: Vec<_> = f.0.borrow().sent.iter().map(|(_, t, m)| (t.clone(), *m)).collect();
|
|
assert_eq!(sent, vec![("첫 문장.".into(), true), (" 둘째 문장.".into(), false)]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_request_the_worker_never_accepted_fails_without_blocking_the_next() {
|
|
let (mut p, f) = setup();
|
|
let a = p.say("a", false);
|
|
let c = p.say("b", false);
|
|
p.done(&done(&a, true));
|
|
p.audio(&c, 0, b("b0"));
|
|
assert_eq!(f.0.borrow().played, vec!["b0"]);
|
|
assert_eq!(p.take_events()[0], end(&a, 0, false, true));
|
|
}
|
|
}
|