The user asked for a faster, real app instead of Electron. The app is now a Rust crate (serenity + songbird); the GUI in M5 will be a Tauri 2 shell on the system WebView2. - Same behaviour as the Electron core: one voice channel, per-speaker PCM to the voice worker, speaking_end and speaker_leave, sentence playback with stop and cancel, transcription notice on join, slash command join/leave/stop (create-or-update), registered chat channels with live events, history and sends that never ping, worker process with a fresh token and restart backoff. - songbird 0.6.1 garbles about a third of received frames under DAVE (its decoder keeps the bytes DAVE decryption leaves at the end of the payload; a 20 ms stepped test tone showed repeats, skips and reordering, while sending was exact). songbird now only decrypts; the core reorders packets in a small window (Opus FEC for the last lost frame) and decodes them with libopus itself. - Live test on real Discord with two bots: five sentences heard (CER 0/0/0/5/12.5%), 0.71-0.90 s from end of speech to final, speech read back at 3.4%, sound gone 0.08 s after stop. The release binary uses about 16 MB of memory; a Windows build (MinGW) logs in under Wine. - Spec moved to app/src/discord_core.spec.md; architecture, decisions, roadmap and pc-features updated. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
272 lines
9.9 KiB
Rust
272 lines
9.9 KiB
Rust
//! Per-speaker receive: RTP packets in arrival order -> Opus frames in sequence order -> PCM.
|
|
//!
|
|
//! songbird 0.6.1 decrypts (transport and DAVE) but feeds its own Opus decoder the payload without
|
|
//! trimming what DAVE decryption left at the end, which garbles or drops about a third of the
|
|
//! frames. The core therefore takes the decrypted packets with their exact payload bounds and
|
|
//! decodes them itself. A small reorder window waits for at most `MAX_WAIT` later packets before a
|
|
//! missing one is concealed (Opus in-band FEC from the next packet when it has it).
|
|
|
|
use std::collections::BTreeMap;
|
|
|
|
/// Later packets to wait for before giving up on a missing one: about 40 ms.
|
|
pub const MAX_WAIT: usize = 2;
|
|
/// A jump this far ahead (or any jump back past it) means the sender restarted its sequence.
|
|
const RESYNC: i16 = 64;
|
|
|
|
pub trait FrameDecoder {
|
|
/// Decode one Opus packet; `fec` decodes the previous (lost) frame from this packet's FEC data.
|
|
fn decode(&mut self, packet: &[u8], fec: bool) -> Option<Vec<i16>>;
|
|
/// Packet-loss concealment for one missing frame.
|
|
fn conceal(&mut self) -> Option<Vec<i16>>;
|
|
}
|
|
|
|
pub struct OpusFrames(opus2::Decoder);
|
|
|
|
impl OpusFrames {
|
|
/// 48 kHz stereo, Discord's format.
|
|
pub fn new() -> Self {
|
|
Self(opus2::Decoder::new(48000, opus2::Channels::Stereo).expect("opus decoder"))
|
|
}
|
|
}
|
|
|
|
impl Default for OpusFrames {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl FrameDecoder for OpusFrames {
|
|
fn decode(&mut self, packet: &[u8], fec: bool) -> Option<Vec<i16>> {
|
|
// FEC must ask for exactly the missing 20 ms; a normal decode gets room for 120 ms, the
|
|
// largest Opus frame. Stereo.
|
|
let mut out = vec![0i16; if fec { 960 * 2 } else { 5760 * 2 }];
|
|
let n = self.0.decode(packet, &mut out, fec).ok()?;
|
|
out.truncate(n * 2);
|
|
Some(out)
|
|
}
|
|
fn conceal(&mut self) -> Option<Vec<i16>> {
|
|
let mut out = vec![0i16; 960 * 2];
|
|
let n = self.0.decode(&[], &mut out, false).ok()?;
|
|
out.truncate(n * 2);
|
|
Some(out)
|
|
}
|
|
}
|
|
|
|
/// One speaker's reorder window and decoder.
|
|
pub struct SpeakerStream<D: FrameDecoder> {
|
|
dec: D,
|
|
next: Option<u16>,
|
|
pending: BTreeMap<i16, Vec<u8>>,
|
|
pub lost: u64,
|
|
pub late: u64,
|
|
pub bad: u64,
|
|
}
|
|
|
|
impl<D: FrameDecoder> SpeakerStream<D> {
|
|
pub fn new(dec: D) -> Self {
|
|
Self { dec, next: None, pending: BTreeMap::new(), lost: 0, late: 0, bad: 0 }
|
|
}
|
|
|
|
/// A packet arrived; returns the PCM frames now ready, in order.
|
|
pub fn push(&mut self, seq: u16, opus: Vec<u8>) -> Vec<Vec<i16>> {
|
|
let mut out = vec![];
|
|
let next = *self.next.get_or_insert(seq);
|
|
let ahead = seq.wrapping_sub(next) as i16;
|
|
if !(-RESYNC..RESYNC).contains(&ahead) {
|
|
// The sender restarted: drop the window and start over at this packet.
|
|
self.flush_into(&mut out);
|
|
self.next = Some(seq);
|
|
self.decode_into(&opus, &mut out);
|
|
self.next = Some(seq.wrapping_add(1));
|
|
return out;
|
|
}
|
|
if ahead < 0 {
|
|
self.late += 1;
|
|
return out;
|
|
}
|
|
self.pending.insert(ahead, opus);
|
|
self.drain(&mut out, false);
|
|
out
|
|
}
|
|
|
|
/// The speaker stopped: play out what is waiting, concealing nothing at the end.
|
|
pub fn flush(&mut self) -> Vec<Vec<i16>> {
|
|
let mut out = vec![];
|
|
self.flush_into(&mut out);
|
|
out
|
|
}
|
|
|
|
fn flush_into(&mut self, out: &mut Vec<Vec<i16>>) {
|
|
self.drain(out, true);
|
|
}
|
|
|
|
fn decode_into(&mut self, opus: &[u8], out: &mut Vec<Vec<i16>>) {
|
|
match self.dec.decode(opus, false) {
|
|
Some(pcm) if !pcm.is_empty() => out.push(pcm),
|
|
_ => self.bad += 1,
|
|
}
|
|
}
|
|
|
|
/// Play every packet that is due; when `force`, or the window holds more than MAX_WAIT packets
|
|
/// past a gap, conceal the gap and go on.
|
|
fn drain(&mut self, out: &mut Vec<Vec<i16>>, force: bool) {
|
|
loop {
|
|
if let Some(opus) = self.pending.remove(&0) {
|
|
self.decode_into(&opus, out);
|
|
self.advance(1);
|
|
continue;
|
|
}
|
|
let Some(&first) = self.pending.keys().next() else { return };
|
|
if !force && self.pending.len() <= MAX_WAIT {
|
|
return;
|
|
}
|
|
// Frames 0..first are missing. Recover the last one from the next packet's FEC, conceal
|
|
// the others.
|
|
for k in 0..first {
|
|
let pcm = if k == first - 1 {
|
|
let next = self.pending.get(&first).cloned().unwrap_or_default();
|
|
self.dec.decode(&next, true)
|
|
} else {
|
|
self.dec.conceal()
|
|
};
|
|
if let Some(pcm) = pcm.filter(|p| !p.is_empty()) {
|
|
out.push(pcm);
|
|
}
|
|
self.lost += 1;
|
|
}
|
|
self.advance(first);
|
|
}
|
|
}
|
|
|
|
fn advance(&mut self, by: i16) {
|
|
self.next = self.next.map(|n| n.wrapping_add(by as u16));
|
|
self.pending = std::mem::take(&mut self.pending).into_iter().map(|(k, v)| (k - by, v)).collect();
|
|
}
|
|
}
|
|
|
|
/// The RTP sequence number and SSRC of a packet, and the Opus payload inside it given songbird's
|
|
/// payload bounds (offset into the body after the fixed header and CSRCs, bytes to drop at the end).
|
|
pub fn opus_payload(packet: &[u8], payload_offset: usize, payload_end_pad: usize) -> Option<(u16, u32, &[u8])> {
|
|
if packet.len() < 12 {
|
|
return None;
|
|
}
|
|
let csrc = usize::from(packet[0] & 0x0f);
|
|
let header = 12 + 4 * csrc;
|
|
let seq = u16::from_be_bytes([packet[2], packet[3]]);
|
|
let ssrc = u32::from_be_bytes([packet[8], packet[9], packet[10], packet[11]]);
|
|
let start = header.checked_add(payload_offset)?;
|
|
let end = packet.len().checked_sub(payload_end_pad)?;
|
|
(start < end).then(|| (seq, ssrc, &packet[start..end]))
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
/// "Decodes" a packet to one sample holding its first byte; FEC and concealment are marked.
|
|
struct Tag;
|
|
impl FrameDecoder for Tag {
|
|
fn decode(&mut self, p: &[u8], fec: bool) -> Option<Vec<i16>> {
|
|
Some(vec![if fec { 1000 + p.first().copied().unwrap_or(0) as i16 } else { p[0] as i16 }])
|
|
}
|
|
fn conceal(&mut self) -> Option<Vec<i16>> {
|
|
Some(vec![-1])
|
|
}
|
|
}
|
|
|
|
fn run(s: &mut SpeakerStream<Tag>, seqs: &[u16]) -> Vec<i16> {
|
|
seqs.iter().flat_map(|&q| s.push(q, vec![q as u8])).map(|f| f[0]).collect()
|
|
}
|
|
|
|
#[test]
|
|
fn in_order_packets_play_at_once() {
|
|
let mut s = SpeakerStream::new(Tag);
|
|
assert_eq!(run(&mut s, &[5, 6, 7]), vec![5, 6, 7]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_swapped_pair_is_put_back_in_order() {
|
|
let mut s = SpeakerStream::new(Tag);
|
|
assert_eq!(run(&mut s, &[1, 3, 2, 4]), vec![1, 2, 3, 4]);
|
|
assert_eq!((s.lost, s.late), (0, 0));
|
|
}
|
|
|
|
#[test]
|
|
fn a_lost_packet_is_recovered_from_fec_after_the_wait() {
|
|
let mut s = SpeakerStream::new(Tag);
|
|
// 3 never comes; after MAX_WAIT + 1 later packets it is rebuilt from 4's FEC data.
|
|
assert_eq!(run(&mut s, &[1, 2, 4, 5]), vec![1, 2]);
|
|
assert_eq!(run(&mut s, &[6]), vec![1004, 4, 5, 6]);
|
|
assert_eq!(s.lost, 1);
|
|
}
|
|
|
|
#[test]
|
|
fn several_lost_packets_are_concealed_then_fec_for_the_last() {
|
|
let mut s = SpeakerStream::new(Tag);
|
|
assert_eq!(run(&mut s, &[1, 4, 5, 6]), vec![1, -1, 1004, 4, 5, 6]);
|
|
assert_eq!(s.lost, 2);
|
|
}
|
|
|
|
#[test]
|
|
fn late_and_duplicate_packets_are_dropped() {
|
|
let mut s = SpeakerStream::new(Tag);
|
|
assert_eq!(run(&mut s, &[1, 2, 3, 2, 1]), vec![1, 2, 3]);
|
|
assert_eq!(s.late, 2);
|
|
}
|
|
|
|
#[test]
|
|
fn sequence_numbers_wrap() {
|
|
let mut s = SpeakerStream::new(Tag);
|
|
let out: Vec<i16> = [65534u16, 65535, 0, 1].iter().flat_map(|&q| s.push(q, vec![(q % 7) as u8])).map(|f| f[0]).collect();
|
|
assert_eq!(out, vec![65534 % 7, 65535 % 7, 0, 1].into_iter().map(|v| v as i16).collect::<Vec<_>>());
|
|
}
|
|
|
|
#[test]
|
|
fn a_far_jump_starts_over() {
|
|
let mut s = SpeakerStream::new(Tag);
|
|
assert_eq!(run(&mut s, &[10, 11, 200, 201]), vec![10, 11, 200, 201]);
|
|
}
|
|
|
|
#[test]
|
|
fn flush_plays_what_waits_behind_a_gap() {
|
|
let mut s = SpeakerStream::new(Tag);
|
|
assert_eq!(run(&mut s, &[1, 3]), vec![1]);
|
|
assert_eq!(s.flush().iter().map(|f| f[0]).collect::<Vec<_>>(), vec![1003, 3]);
|
|
}
|
|
|
|
#[test]
|
|
fn real_opus_round_trip_through_the_window() {
|
|
// Encode 5 frames of a tone, drop one, swap two: all 5 frames come out, in order.
|
|
let mut enc = opus2::Encoder::new(48000, opus2::Channels::Stereo, opus2::Application::Voip).unwrap();
|
|
enc.set_inband_fec(true).unwrap();
|
|
enc.set_packet_loss_perc(10).unwrap();
|
|
let frames: Vec<Vec<u8>> = (0..6).map(|k| {
|
|
let pcm: Vec<i16> = (0..960).flat_map(|i| {
|
|
let v = (8000.0 * (2.0 * std::f64::consts::PI * 440.0 * (k * 960 + i) as f64 / 48000.0).sin()) as i16;
|
|
[v, v]
|
|
}).collect();
|
|
enc.encode_vec(&pcm, 4000).unwrap()
|
|
}).collect();
|
|
let mut s = SpeakerStream::new(OpusFrames::new());
|
|
let mut out = vec![];
|
|
for q in [0u16, 2, 1, 4, 5] {
|
|
out.extend(s.push(q, frames[q as usize].clone()));
|
|
}
|
|
out.extend(s.flush());
|
|
assert_eq!(out.len(), 6);
|
|
assert!(out.iter().all(|f| f.len() == 1920));
|
|
assert_eq!(s.lost, 1);
|
|
}
|
|
|
|
#[test]
|
|
fn payload_bounds_follow_the_header_and_songbirds_offsets() {
|
|
let mut p = vec![0x80u8, 0x78, 0x01, 0x02, 0, 0, 0, 0, 0, 0, 0x30, 0x39];
|
|
p.extend_from_slice(&[9, 9, 9, 9]); // extension (offset 4)
|
|
p.extend_from_slice(&[1, 2, 3]); // opus
|
|
p.extend_from_slice(&[7, 7]); // DAVE leftovers (end pad 2)
|
|
let (seq, ssrc, opus) = opus_payload(&p, 4, 2).unwrap();
|
|
assert_eq!((seq, ssrc, opus), (0x0102, 12345, &[1u8, 2, 3][..]));
|
|
assert!(opus_payload(&p, 10, 9).is_none());
|
|
}
|
|
}
|