Files
javis_bot/app/src/playback.rs
javis-bot f25c26cf53 refactor(app): rewrite the M2 Discord voice core in Rust
The user asked for a faster, real app instead of Electron. The app is now a Rust crate (serenity +
songbird); the GUI in M5 will be a Tauri 2 shell on the system WebView2.

- Same behaviour as the Electron core: one voice channel, per-speaker PCM to the voice worker,
  speaking_end and speaker_leave, sentence playback with stop and cancel, transcription notice on
  join, slash command join/leave/stop (create-or-update), registered chat channels with live events,
  history and sends that never ping, worker process with a fresh token and restart backoff.
- songbird 0.6.1 garbles about a third of received frames under DAVE (its decoder keeps the bytes
  DAVE decryption leaves at the end of the payload; a 20 ms stepped test tone showed repeats, skips
  and reordering, while sending was exact). songbird now only decrypts; the core reorders packets in
  a small window (Opus FEC for the last lost frame) and decodes them with libopus itself.
- Live test on real Discord with two bots: five sentences heard (CER 0/0/0/5/12.5%), 0.71-0.90 s
  from end of speech to final, speech read back at 3.4%, sound gone 0.08 s after stop. The release
  binary uses about 16 MB of memory; a Windows build (MinGW) logs in under Wine.
- Spec moved to app/src/discord_core.spec.md; architecture, decisions, roadmap and pc-features updated.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 11:57:53 +09:00

316 lines
11 KiB
Rust

//! The bot's mouth: turns text into queued speech, one request after another.
//!
//! Each request goes to the voice worker as TTS; the worker sends back one PCM chunk per sentence.
//! Chunks play in order as soon as they arrive, so the first sentence plays while later ones are still
//! being made. Requests play first come, first served. `stop()` silences everything at once and
//! cancels what the worker has not made yet.
use std::collections::{BTreeMap, VecDeque};
use crate::worker_client::TtsDone;
pub trait TtsBackend {
/// Send text for request `id`; false when the worker is not connected (the request then fails).
fn tts(&self, id: &str, text: &str, more: bool) -> bool;
fn tts_cancel(&self, id: &str);
}
/// Plays one PCM clip at a time; its owner calls `Playback::clip_ended()` when the clip finishes.
pub trait ClipSink {
fn play(&mut self, pcm: Vec<u8>);
fn stop(&mut self);
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PlaybackEnd {
pub id: String,
/// Sentences that were played.
pub played: u32,
pub cancelled: bool,
pub failed: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PlaybackEvent {
Start(String),
End(PlaybackEnd),
}
struct Request {
id: String,
chunks: BTreeMap<u32, Vec<u8>>,
next: u32,
done: bool,
failed: bool,
played: u32,
started: bool,
}
pub struct Playback<B: TtsBackend, S: ClipSink> {
backend: B,
sink: S,
queue: VecDeque<Request>,
playing: bool,
counter: u64,
events: Vec<PlaybackEvent>,
}
impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
pub fn new(backend: B, sink: S) -> Self {
Self { backend, sink, queue: VecDeque::new(), playing: false, counter: 0, events: vec![] }
}
/// True while a clip is playing or a request is waiting for its audio.
pub fn busy(&self) -> bool {
self.playing || !self.queue.is_empty()
}
/// Start and end events since the last call.
pub fn take_events(&mut self) -> Vec<PlaybackEvent> {
std::mem::take(&mut self.events)
}
/// Queue `text` as a new request; with `more`, add text later with `continue_text`. Returns its id.
pub fn say(&mut self, text: &str, more: bool) -> String {
self.counter += 1;
let id = format!("r{}", self.counter);
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false,
played: 0, started: false });
crate::debug_log!("play", "say id={id} chars={} more={more} queued={}", text.chars().count(), self.queue.len());
self.backend.tts(&id, text, more);
id
}
/// More text for an open request (an LLM answer that is still streaming).
pub fn continue_text(&mut self, id: &str, text: &str, more: bool) {
if self.queue.iter().any(|r| r.id == id && !r.done) {
self.backend.tts(id, text, more);
}
}
/// A sentence of audio arrived from the worker.
pub fn audio(&mut self, id: &str, seq: u32, pcm: Vec<u8>) {
if let Some(r) = self.queue.iter_mut().find(|r| r.id == id) {
r.chunks.insert(seq, pcm);
self.pump();
}
}
/// The worker finished a request (all sentences sent, cancelled, or failed).
pub fn done(&mut self, d: &TtsDone) {
if let Some(r) = self.queue.iter_mut().find(|r| r.id == d.id) {
r.done = true;
r.failed = d.failed;
self.pump();
}
}
/// The sink finished playing the current clip.
pub fn clip_ended(&mut self) {
self.playing = false;
self.pump();
}
/// Silence now: stop the clip, drop every queued request, cancel the worker's work.
pub fn stop(&mut self) -> usize {
let stopped: Vec<Request> = self.queue.drain(..).collect();
let was_playing = self.playing;
self.playing = false;
for r in stopped.iter().filter(|r| !r.done) {
self.backend.tts_cancel(&r.id);
}
if was_playing {
self.sink.stop();
}
for r in &stopped {
self.events.push(PlaybackEvent::End(PlaybackEnd { id: r.id.clone(), played: r.played, cancelled: true,
failed: false }));
}
crate::debug_log!("play", "stop requests={} was_playing={was_playing}", stopped.len());
stopped.len()
}
fn pump(&mut self) {
while !self.playing {
let Some(head) = self.queue.front_mut() else { return };
if !head.chunks.contains_key(&head.next) && head.done {
// A sentence failed: skip its gap and play what did arrive.
if let Some(&first) = head.chunks.keys().next() {
head.next = first;
}
}
if let Some(pcm) = head.chunks.remove(&head.next) {
head.next += 1;
head.played += 1;
if !head.started {
head.started = true;
self.events.push(PlaybackEvent::Start(head.id.clone()));
}
self.playing = true;
self.sink.play(pcm);
return;
}
if !head.done {
return;
}
let r = self.queue.pop_front().expect("head exists");
crate::debug_log!("play", "said id={} sentences={} failed={}", r.id, r.played, r.failed);
self.events.push(PlaybackEvent::End(PlaybackEnd { id: r.id, played: r.played, cancelled: false,
failed: r.failed }));
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::{cell::RefCell, rc::Rc};
#[derive(Default)]
struct Log {
sent: Vec<(String, String, bool)>,
cancelled: Vec<String>,
played: Vec<String>,
stops: usize,
}
#[derive(Clone, Default)]
struct Fake(Rc<RefCell<Log>>);
impl TtsBackend for Fake {
fn tts(&self, id: &str, text: &str, more: bool) -> bool {
self.0.borrow_mut().sent.push((id.into(), text.into(), more));
true
}
fn tts_cancel(&self, id: &str) {
self.0.borrow_mut().cancelled.push(id.into());
}
}
impl ClipSink for Fake {
fn play(&mut self, pcm: Vec<u8>) {
self.0.borrow_mut().played.push(String::from_utf8(pcm).unwrap());
}
fn stop(&mut self) {
self.0.borrow_mut().stops += 1;
}
}
fn setup() -> (Playback<Fake, Fake>, Fake) {
let f = Fake::default();
(Playback::new(f.clone(), f.clone()), f)
}
fn done(id: &str, failed: bool) -> TtsDone {
TtsDone { id: id.into(), chunks: 0, cancelled: false, failed }
}
fn end(id: &str, played: u32, cancelled: bool, failed: bool) -> PlaybackEvent {
PlaybackEvent::End(PlaybackEnd { id: id.into(), played, cancelled, failed })
}
fn b(s: &str) -> Vec<u8> {
s.as_bytes().to_vec()
}
#[test]
fn plays_the_first_sentence_on_arrival_and_the_next_when_the_clip_ends() {
let (mut p, f) = setup();
let id = p.say("하나. 둘.", false);
assert_eq!(f.0.borrow().sent, vec![(id.clone(), "하나. 둘.".into(), false)]);
p.audio(&id, 0, b("s0"));
assert_eq!(f.0.borrow().played, vec!["s0"]);
p.audio(&id, 1, b("s1"));
assert_eq!(f.0.borrow().played, vec!["s0"]);
p.clip_ended();
assert_eq!(f.0.borrow().played, vec!["s0", "s1"]);
p.done(&done(&id, false));
assert_eq!(p.take_events(), vec![PlaybackEvent::Start(id.clone())]);
p.clip_ended();
assert_eq!(p.take_events(), vec![end(&id, 2, false, false)]);
assert!(!p.busy());
}
#[test]
fn requests_play_in_the_order_they_were_made() {
let (mut p, f) = setup();
let a = p.say("a", false);
let c = p.say("b", false);
p.audio(&c, 0, b("b0"));
assert!(f.0.borrow().played.is_empty());
p.audio(&a, 0, b("a0"));
p.done(&done(&a, false));
p.done(&done(&c, false));
p.clip_ended();
p.clip_ended();
assert_eq!(f.0.borrow().played, vec!["a0", "b0"]);
let ends: Vec<_> = p.take_events().into_iter().filter_map(|e| match e { PlaybackEvent::End(e) => Some(e.id), _ => None }).collect();
assert_eq!(ends, vec![a, c]);
}
#[test]
fn sentences_stay_in_order_when_they_arrive_out_of_order() {
let (mut p, f) = setup();
let id = p.say("x", true);
p.audio(&id, 1, b("s1"));
assert!(f.0.borrow().played.is_empty());
p.audio(&id, 0, b("s0"));
p.clip_ended();
assert_eq!(f.0.borrow().played, vec!["s0", "s1"]);
}
#[test]
fn a_failed_sentence_is_skipped_and_the_rest_still_play() {
let (mut p, f) = setup();
let id = p.say("a. b. c.", false);
p.audio(&id, 0, b("s0"));
p.audio(&id, 2, b("s2"));
p.done(&done(&id, true));
p.clip_ended();
p.clip_ended();
assert_eq!(f.0.borrow().played, vec!["s0", "s2"]);
assert_eq!(p.take_events().last(), Some(&end(&id, 2, false, true)));
}
#[test]
fn stop_silences_cancels_unfinished_and_drops_the_queue() {
let (mut p, f) = setup();
let a = p.say("a", true);
let c = p.say("b", false);
p.audio(&a, 0, b("a0"));
p.done(&done(&c, false));
p.take_events();
assert_eq!(p.stop(), 2);
assert_eq!(f.0.borrow().stops, 1);
assert_eq!(f.0.borrow().cancelled, vec![a.clone()]);
assert_eq!(p.take_events(), vec![end(&a, 1, true, false), end(&c, 0, true, false)]);
p.audio(&a, 1, b("a1"));
p.clip_ended();
assert_eq!(f.0.borrow().played, vec!["a0"]);
assert!(!p.busy());
}
#[test]
fn stop_with_nothing_to_say_touches_nothing() {
let (mut p, f) = setup();
assert_eq!(p.stop(), 0);
assert_eq!(f.0.borrow().stops, 0);
}
#[test]
fn text_can_be_added_to_an_open_request_but_not_a_finished_one() {
let (mut p, f) = setup();
let id = p.say("첫 문장.", true);
p.continue_text(&id, " 둘째 문장.", false);
p.done(&done(&id, false));
p.continue_text(&id, "늦은 글", false);
let sent: Vec<_> = f.0.borrow().sent.iter().map(|(_, t, m)| (t.clone(), *m)).collect();
assert_eq!(sent, vec![("첫 문장.".into(), true), (" 둘째 문장.".into(), false)]);
}
#[test]
fn a_request_the_worker_never_accepted_fails_without_blocking_the_next() {
let (mut p, f) = setup();
let a = p.say("a", false);
let c = p.say("b", false);
p.done(&done(&a, true));
p.audio(&c, 0, b("b0"));
assert_eq!(f.0.borrow().played, vec!["b0"]);
assert_eq!(p.take_events()[0], end(&a, 0, false, true));
}
}