Files
javis_bot/app/src/receive.rs
javis-bot f25c26cf53 refactor(app): rewrite the M2 Discord voice core in Rust
The user asked for a faster, real app instead of Electron. The app is now a Rust crate (serenity +
songbird); the GUI in M5 will be a Tauri 2 shell on the system WebView2.

- Same behaviour as the Electron core: one voice channel, per-speaker PCM to the voice worker,
  speaking_end and speaker_leave, sentence playback with stop and cancel, transcription notice on
  join, slash command join/leave/stop (create-or-update), registered chat channels with live events,
  history and sends that never ping, worker process with a fresh token and restart backoff.
- songbird 0.6.1 garbles about a third of received frames under DAVE (its decoder keeps the bytes
  DAVE decryption leaves at the end of the payload; a 20 ms stepped test tone showed repeats, skips
  and reordering, while sending was exact). songbird now only decrypts; the core reorders packets in
  a small window (Opus FEC for the last lost frame) and decodes them with libopus itself.
- Live test on real Discord with two bots: five sentences heard (CER 0/0/0/5/12.5%), 0.71-0.90 s
  from end of speech to final, speech read back at 3.4%, sound gone 0.08 s after stop. The release
  binary uses about 16 MB of memory; a Windows build (MinGW) logs in under Wine.
- Spec moved to app/src/discord_core.spec.md; architecture, decisions, roadmap and pc-features updated.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 11:57:53 +09:00

272 lines
9.9 KiB
Rust

//! Per-speaker receive: RTP packets in arrival order -> Opus frames in sequence order -> PCM.
//!
//! songbird 0.6.1 decrypts (transport and DAVE) but feeds its own Opus decoder the payload without
//! trimming what DAVE decryption left at the end, which garbles or drops about a third of the
//! frames. The core therefore takes the decrypted packets with their exact payload bounds and
//! decodes them itself. A small reorder window waits for at most `MAX_WAIT` later packets before a
//! missing one is concealed (Opus in-band FEC from the next packet when it has it).
use std::collections::BTreeMap;
/// Later packets to wait for before giving up on a missing one: about 40 ms.
pub const MAX_WAIT: usize = 2;
/// A jump this far ahead (or any jump back past it) means the sender restarted its sequence.
const RESYNC: i16 = 64;
pub trait FrameDecoder {
/// Decode one Opus packet; `fec` decodes the previous (lost) frame from this packet's FEC data.
fn decode(&mut self, packet: &[u8], fec: bool) -> Option<Vec<i16>>;
/// Packet-loss concealment for one missing frame.
fn conceal(&mut self) -> Option<Vec<i16>>;
}
pub struct OpusFrames(opus2::Decoder);
impl OpusFrames {
/// 48 kHz stereo, Discord's format.
pub fn new() -> Self {
Self(opus2::Decoder::new(48000, opus2::Channels::Stereo).expect("opus decoder"))
}
}
impl Default for OpusFrames {
fn default() -> Self {
Self::new()
}
}
impl FrameDecoder for OpusFrames {
fn decode(&mut self, packet: &[u8], fec: bool) -> Option<Vec<i16>> {
// FEC must ask for exactly the missing 20 ms; a normal decode gets room for 120 ms, the
// largest Opus frame. Stereo.
let mut out = vec![0i16; if fec { 960 * 2 } else { 5760 * 2 }];
let n = self.0.decode(packet, &mut out, fec).ok()?;
out.truncate(n * 2);
Some(out)
}
fn conceal(&mut self) -> Option<Vec<i16>> {
let mut out = vec![0i16; 960 * 2];
let n = self.0.decode(&[], &mut out, false).ok()?;
out.truncate(n * 2);
Some(out)
}
}
/// One speaker's reorder window and decoder.
pub struct SpeakerStream<D: FrameDecoder> {
dec: D,
next: Option<u16>,
pending: BTreeMap<i16, Vec<u8>>,
pub lost: u64,
pub late: u64,
pub bad: u64,
}
impl<D: FrameDecoder> SpeakerStream<D> {
pub fn new(dec: D) -> Self {
Self { dec, next: None, pending: BTreeMap::new(), lost: 0, late: 0, bad: 0 }
}
/// A packet arrived; returns the PCM frames now ready, in order.
pub fn push(&mut self, seq: u16, opus: Vec<u8>) -> Vec<Vec<i16>> {
let mut out = vec![];
let next = *self.next.get_or_insert(seq);
let ahead = seq.wrapping_sub(next) as i16;
if !(-RESYNC..RESYNC).contains(&ahead) {
// The sender restarted: drop the window and start over at this packet.
self.flush_into(&mut out);
self.next = Some(seq);
self.decode_into(&opus, &mut out);
self.next = Some(seq.wrapping_add(1));
return out;
}
if ahead < 0 {
self.late += 1;
return out;
}
self.pending.insert(ahead, opus);
self.drain(&mut out, false);
out
}
/// The speaker stopped: play out what is waiting, concealing nothing at the end.
pub fn flush(&mut self) -> Vec<Vec<i16>> {
let mut out = vec![];
self.flush_into(&mut out);
out
}
fn flush_into(&mut self, out: &mut Vec<Vec<i16>>) {
self.drain(out, true);
}
fn decode_into(&mut self, opus: &[u8], out: &mut Vec<Vec<i16>>) {
match self.dec.decode(opus, false) {
Some(pcm) if !pcm.is_empty() => out.push(pcm),
_ => self.bad += 1,
}
}
/// Play every packet that is due; when `force`, or the window holds more than MAX_WAIT packets
/// past a gap, conceal the gap and go on.
fn drain(&mut self, out: &mut Vec<Vec<i16>>, force: bool) {
loop {
if let Some(opus) = self.pending.remove(&0) {
self.decode_into(&opus, out);
self.advance(1);
continue;
}
let Some(&first) = self.pending.keys().next() else { return };
if !force && self.pending.len() <= MAX_WAIT {
return;
}
// Frames 0..first are missing. Recover the last one from the next packet's FEC, conceal
// the others.
for k in 0..first {
let pcm = if k == first - 1 {
let next = self.pending.get(&first).cloned().unwrap_or_default();
self.dec.decode(&next, true)
} else {
self.dec.conceal()
};
if let Some(pcm) = pcm.filter(|p| !p.is_empty()) {
out.push(pcm);
}
self.lost += 1;
}
self.advance(first);
}
}
fn advance(&mut self, by: i16) {
self.next = self.next.map(|n| n.wrapping_add(by as u16));
self.pending = std::mem::take(&mut self.pending).into_iter().map(|(k, v)| (k - by, v)).collect();
}
}
/// The RTP sequence number and SSRC of a packet, and the Opus payload inside it given songbird's
/// payload bounds (offset into the body after the fixed header and CSRCs, bytes to drop at the end).
pub fn opus_payload(packet: &[u8], payload_offset: usize, payload_end_pad: usize) -> Option<(u16, u32, &[u8])> {
if packet.len() < 12 {
return None;
}
let csrc = usize::from(packet[0] & 0x0f);
let header = 12 + 4 * csrc;
let seq = u16::from_be_bytes([packet[2], packet[3]]);
let ssrc = u32::from_be_bytes([packet[8], packet[9], packet[10], packet[11]]);
let start = header.checked_add(payload_offset)?;
let end = packet.len().checked_sub(payload_end_pad)?;
(start < end).then(|| (seq, ssrc, &packet[start..end]))
}
#[cfg(test)]
mod tests {
use super::*;
/// "Decodes" a packet to one sample holding its first byte; FEC and concealment are marked.
struct Tag;
impl FrameDecoder for Tag {
fn decode(&mut self, p: &[u8], fec: bool) -> Option<Vec<i16>> {
Some(vec![if fec { 1000 + p.first().copied().unwrap_or(0) as i16 } else { p[0] as i16 }])
}
fn conceal(&mut self) -> Option<Vec<i16>> {
Some(vec![-1])
}
}
fn run(s: &mut SpeakerStream<Tag>, seqs: &[u16]) -> Vec<i16> {
seqs.iter().flat_map(|&q| s.push(q, vec![q as u8])).map(|f| f[0]).collect()
}
#[test]
fn in_order_packets_play_at_once() {
let mut s = SpeakerStream::new(Tag);
assert_eq!(run(&mut s, &[5, 6, 7]), vec![5, 6, 7]);
}
#[test]
fn a_swapped_pair_is_put_back_in_order() {
let mut s = SpeakerStream::new(Tag);
assert_eq!(run(&mut s, &[1, 3, 2, 4]), vec![1, 2, 3, 4]);
assert_eq!((s.lost, s.late), (0, 0));
}
#[test]
fn a_lost_packet_is_recovered_from_fec_after_the_wait() {
let mut s = SpeakerStream::new(Tag);
// 3 never comes; after MAX_WAIT + 1 later packets it is rebuilt from 4's FEC data.
assert_eq!(run(&mut s, &[1, 2, 4, 5]), vec![1, 2]);
assert_eq!(run(&mut s, &[6]), vec![1004, 4, 5, 6]);
assert_eq!(s.lost, 1);
}
#[test]
fn several_lost_packets_are_concealed_then_fec_for_the_last() {
let mut s = SpeakerStream::new(Tag);
assert_eq!(run(&mut s, &[1, 4, 5, 6]), vec![1, -1, 1004, 4, 5, 6]);
assert_eq!(s.lost, 2);
}
#[test]
fn late_and_duplicate_packets_are_dropped() {
let mut s = SpeakerStream::new(Tag);
assert_eq!(run(&mut s, &[1, 2, 3, 2, 1]), vec![1, 2, 3]);
assert_eq!(s.late, 2);
}
#[test]
fn sequence_numbers_wrap() {
let mut s = SpeakerStream::new(Tag);
let out: Vec<i16> = [65534u16, 65535, 0, 1].iter().flat_map(|&q| s.push(q, vec![(q % 7) as u8])).map(|f| f[0]).collect();
assert_eq!(out, vec![65534 % 7, 65535 % 7, 0, 1].into_iter().map(|v| v as i16).collect::<Vec<_>>());
}
#[test]
fn a_far_jump_starts_over() {
let mut s = SpeakerStream::new(Tag);
assert_eq!(run(&mut s, &[10, 11, 200, 201]), vec![10, 11, 200, 201]);
}
#[test]
fn flush_plays_what_waits_behind_a_gap() {
let mut s = SpeakerStream::new(Tag);
assert_eq!(run(&mut s, &[1, 3]), vec![1]);
assert_eq!(s.flush().iter().map(|f| f[0]).collect::<Vec<_>>(), vec![1003, 3]);
}
#[test]
fn real_opus_round_trip_through_the_window() {
// Encode 5 frames of a tone, drop one, swap two: all 5 frames come out, in order.
let mut enc = opus2::Encoder::new(48000, opus2::Channels::Stereo, opus2::Application::Voip).unwrap();
enc.set_inband_fec(true).unwrap();
enc.set_packet_loss_perc(10).unwrap();
let frames: Vec<Vec<u8>> = (0..6).map(|k| {
let pcm: Vec<i16> = (0..960).flat_map(|i| {
let v = (8000.0 * (2.0 * std::f64::consts::PI * 440.0 * (k * 960 + i) as f64 / 48000.0).sin()) as i16;
[v, v]
}).collect();
enc.encode_vec(&pcm, 4000).unwrap()
}).collect();
let mut s = SpeakerStream::new(OpusFrames::new());
let mut out = vec![];
for q in [0u16, 2, 1, 4, 5] {
out.extend(s.push(q, frames[q as usize].clone()));
}
out.extend(s.flush());
assert_eq!(out.len(), 6);
assert!(out.iter().all(|f| f.len() == 1920));
assert_eq!(s.lost, 1);
}
#[test]
fn payload_bounds_follow_the_header_and_songbirds_offsets() {
let mut p = vec![0x80u8, 0x78, 0x01, 0x02, 0, 0, 0, 0, 0, 0, 0x30, 0x39];
p.extend_from_slice(&[9, 9, 9, 9]); // extension (offset 4)
p.extend_from_slice(&[1, 2, 3]); // opus
p.extend_from_slice(&[7, 7]); // DAVE leftovers (end pad 2)
let (seq, ssrc, opus) = opus_payload(&p, 4, 2).unwrap();
assert_eq!((seq, ssrc, opus), (0x0102, 12345, &[1u8, 2, 3][..]));
assert!(opus_payload(&p, 10, 9).is_none());
}
}