//! Per-speaker receive: RTP packets in arrival order -> Opus frames in sequence order -> PCM. //! //! songbird 0.6.1 decrypts (transport and DAVE) but feeds its own Opus decoder the payload without //! trimming what DAVE decryption left at the end, which garbles or drops about a third of the //! frames. The core therefore takes the decrypted packets with their exact payload bounds and //! decodes them itself. A small reorder window waits for at most `MAX_WAIT` later packets before a //! missing one is concealed (Opus in-band FEC from the next packet when it has it). use std::collections::BTreeMap; /// Later packets to wait for before giving up on a missing one: about 40 ms. pub const MAX_WAIT: usize = 2; /// A jump this far ahead (or any jump back past it) means the sender restarted its sequence. const RESYNC: i16 = 64; pub trait FrameDecoder { /// Decode one Opus packet; `fec` decodes the previous (lost) frame from this packet's FEC data. fn decode(&mut self, packet: &[u8], fec: bool) -> Option>; /// Packet-loss concealment for one missing frame. fn conceal(&mut self) -> Option>; } pub struct OpusFrames(opus2::Decoder); impl OpusFrames { /// 48 kHz stereo, Discord's format. pub fn new() -> Self { Self(opus2::Decoder::new(48000, opus2::Channels::Stereo).expect("opus decoder")) } } impl Default for OpusFrames { fn default() -> Self { Self::new() } } impl FrameDecoder for OpusFrames { fn decode(&mut self, packet: &[u8], fec: bool) -> Option> { // FEC must ask for exactly the missing 20 ms; a normal decode gets room for 120 ms, the // largest Opus frame. Stereo. let mut out = vec![0i16; if fec { 960 * 2 } else { 5760 * 2 }]; let n = self.0.decode(packet, &mut out, fec).ok()?; out.truncate(n * 2); Some(out) } fn conceal(&mut self) -> Option> { let mut out = vec![0i16; 960 * 2]; let n = self.0.decode(&[], &mut out, false).ok()?; out.truncate(n * 2); Some(out) } } /// One speaker's reorder window and decoder. pub struct SpeakerStream { dec: D, next: Option, pending: BTreeMap>, pub lost: u64, pub late: u64, pub bad: u64, } impl SpeakerStream { pub fn new(dec: D) -> Self { Self { dec, next: None, pending: BTreeMap::new(), lost: 0, late: 0, bad: 0 } } /// A packet arrived; returns the PCM frames now ready, in order. pub fn push(&mut self, seq: u16, opus: Vec) -> Vec> { let mut out = vec![]; let next = *self.next.get_or_insert(seq); let ahead = seq.wrapping_sub(next) as i16; if !(-RESYNC..RESYNC).contains(&ahead) { // The sender restarted: drop the window and start over at this packet. self.flush_into(&mut out); self.next = Some(seq); self.decode_into(&opus, &mut out); self.next = Some(seq.wrapping_add(1)); return out; } if ahead < 0 { self.late += 1; return out; } self.pending.insert(ahead, opus); self.drain(&mut out, false); out } /// The speaker stopped: play out what is waiting, concealing nothing at the end. pub fn flush(&mut self) -> Vec> { let mut out = vec![]; self.flush_into(&mut out); out } fn flush_into(&mut self, out: &mut Vec>) { self.drain(out, true); } fn decode_into(&mut self, opus: &[u8], out: &mut Vec>) { match self.dec.decode(opus, false) { Some(pcm) if !pcm.is_empty() => out.push(pcm), _ => self.bad += 1, } } /// Play every packet that is due; when `force`, or the window holds more than MAX_WAIT packets /// past a gap, conceal the gap and go on. fn drain(&mut self, out: &mut Vec>, force: bool) { loop { if let Some(opus) = self.pending.remove(&0) { self.decode_into(&opus, out); self.advance(1); continue; } let Some(&first) = self.pending.keys().next() else { return }; if !force && self.pending.len() <= MAX_WAIT { return; } // Frames 0..first are missing. Recover the last one from the next packet's FEC, conceal // the others. for k in 0..first { let pcm = if k == first - 1 { let next = self.pending.get(&first).cloned().unwrap_or_default(); self.dec.decode(&next, true) } else { self.dec.conceal() }; if let Some(pcm) = pcm.filter(|p| !p.is_empty()) { out.push(pcm); } self.lost += 1; } self.advance(first); } } fn advance(&mut self, by: i16) { self.next = self.next.map(|n| n.wrapping_add(by as u16)); self.pending = std::mem::take(&mut self.pending).into_iter().map(|(k, v)| (k - by, v)).collect(); } } /// The RTP sequence number and SSRC of a packet, and the Opus payload inside it given songbird's /// payload bounds (offset into the body after the fixed header and CSRCs, bytes to drop at the end). pub fn opus_payload(packet: &[u8], payload_offset: usize, payload_end_pad: usize) -> Option<(u16, u32, &[u8])> { if packet.len() < 12 { return None; } let csrc = usize::from(packet[0] & 0x0f); let header = 12 + 4 * csrc; let seq = u16::from_be_bytes([packet[2], packet[3]]); let ssrc = u32::from_be_bytes([packet[8], packet[9], packet[10], packet[11]]); let start = header.checked_add(payload_offset)?; let end = packet.len().checked_sub(payload_end_pad)?; (start < end).then(|| (seq, ssrc, &packet[start..end])) } #[cfg(test)] mod tests { use super::*; /// "Decodes" a packet to one sample holding its first byte; FEC and concealment are marked. struct Tag; impl FrameDecoder for Tag { fn decode(&mut self, p: &[u8], fec: bool) -> Option> { Some(vec![if fec { 1000 + p.first().copied().unwrap_or(0) as i16 } else { p[0] as i16 }]) } fn conceal(&mut self) -> Option> { Some(vec![-1]) } } fn run(s: &mut SpeakerStream, seqs: &[u16]) -> Vec { seqs.iter().flat_map(|&q| s.push(q, vec![q as u8])).map(|f| f[0]).collect() } #[test] fn in_order_packets_play_at_once() { let mut s = SpeakerStream::new(Tag); assert_eq!(run(&mut s, &[5, 6, 7]), vec![5, 6, 7]); } #[test] fn a_swapped_pair_is_put_back_in_order() { let mut s = SpeakerStream::new(Tag); assert_eq!(run(&mut s, &[1, 3, 2, 4]), vec![1, 2, 3, 4]); assert_eq!((s.lost, s.late), (0, 0)); } #[test] fn a_lost_packet_is_recovered_from_fec_after_the_wait() { let mut s = SpeakerStream::new(Tag); // 3 never comes; after MAX_WAIT + 1 later packets it is rebuilt from 4's FEC data. assert_eq!(run(&mut s, &[1, 2, 4, 5]), vec![1, 2]); assert_eq!(run(&mut s, &[6]), vec![1004, 4, 5, 6]); assert_eq!(s.lost, 1); } #[test] fn several_lost_packets_are_concealed_then_fec_for_the_last() { let mut s = SpeakerStream::new(Tag); assert_eq!(run(&mut s, &[1, 4, 5, 6]), vec![1, -1, 1004, 4, 5, 6]); assert_eq!(s.lost, 2); } #[test] fn late_and_duplicate_packets_are_dropped() { let mut s = SpeakerStream::new(Tag); assert_eq!(run(&mut s, &[1, 2, 3, 2, 1]), vec![1, 2, 3]); assert_eq!(s.late, 2); } #[test] fn sequence_numbers_wrap() { let mut s = SpeakerStream::new(Tag); let out: Vec = [65534u16, 65535, 0, 1].iter().flat_map(|&q| s.push(q, vec![(q % 7) as u8])).map(|f| f[0]).collect(); assert_eq!(out, vec![65534 % 7, 65535 % 7, 0, 1].into_iter().map(|v| v as i16).collect::>()); } #[test] fn a_far_jump_starts_over() { let mut s = SpeakerStream::new(Tag); assert_eq!(run(&mut s, &[10, 11, 200, 201]), vec![10, 11, 200, 201]); } #[test] fn flush_plays_what_waits_behind_a_gap() { let mut s = SpeakerStream::new(Tag); assert_eq!(run(&mut s, &[1, 3]), vec![1]); assert_eq!(s.flush().iter().map(|f| f[0]).collect::>(), vec![1003, 3]); } #[test] fn real_opus_round_trip_through_the_window() { // Encode 5 frames of a tone, drop one, swap two: all 5 frames come out, in order. let mut enc = opus2::Encoder::new(48000, opus2::Channels::Stereo, opus2::Application::Voip).unwrap(); enc.set_inband_fec(true).unwrap(); enc.set_packet_loss_perc(10).unwrap(); let frames: Vec> = (0..6).map(|k| { let pcm: Vec = (0..960).flat_map(|i| { let v = (8000.0 * (2.0 * std::f64::consts::PI * 440.0 * (k * 960 + i) as f64 / 48000.0).sin()) as i16; [v, v] }).collect(); enc.encode_vec(&pcm, 4000).unwrap() }).collect(); let mut s = SpeakerStream::new(OpusFrames::new()); let mut out = vec![]; for q in [0u16, 2, 1, 4, 5] { out.extend(s.push(q, frames[q as usize].clone())); } out.extend(s.flush()); assert_eq!(out.len(), 6); assert!(out.iter().all(|f| f.len() == 1920)); assert_eq!(s.lost, 1); } #[test] fn payload_bounds_follow_the_header_and_songbirds_offsets() { let mut p = vec![0x80u8, 0x78, 0x01, 0x02, 0, 0, 0, 0, 0, 0, 0x30, 0x39]; p.extend_from_slice(&[9, 9, 9, 9]); // extension (offset 4) p.extend_from_slice(&[1, 2, 3]); // opus p.extend_from_slice(&[7, 7]); // DAVE leftovers (end pad 2) let (seq, ssrc, opus) = opus_payload(&p, 4, 2).unwrap(); assert_eq!((seq, ssrc, opus), (0x0102, 12345, &[1u8, 2, 3][..])); assert!(opus_payload(&p, 10, 9).is_none()); } }