fix(voice): release held packets within 20 ms once the speaker is named
Held packets were only processed before the same SSRC's next packet, so a short first utterance that ended before Discord's Speaking op came was dropped after a second. The UDP task now also releases every newly named SSRC's held packets on its 20 ms playout tick. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -140,7 +140,8 @@ and the late "left" update cancelled the new join (seen on real Discord).
|
||||
it could not yet decrypt, which cut the first ~100 ms off the first thing each person said after a
|
||||
join (live 2026-11-18: "자비스" heard as "아비"). The app therefore uses songbird 0.6.1 with a patch
|
||||
(`app/vendor/songbird`, `JARVIS-PATCH.md`): packets of an SSRC not named yet are held, at most one
|
||||
second and 50 packets for each of at most 32 SSRCs, and processed in arrival order once it is named;
|
||||
second and 50 packets for each of at most 32 SSRCs, and processed in arrival order once it is named
|
||||
(before its next packet, or within 20 ms when none follows, as after a short first utterance);
|
||||
the Speaking event goes out first, so the core knows the speaker before their held packets come.
|
||||
The core takes each decrypted RTP packet with songbird's exact payload bounds:
|
||||
- Packets go through a per-speaker reorder window in sequence-number order. A missing packet is
|
||||
|
||||
5
app/vendor/songbird/JARVIS-PATCH.md
vendored
5
app/vendor/songbird/JARVIS-PATCH.md
vendored
@@ -4,8 +4,9 @@ This is songbird 0.6.1 from crates.io (https://github.com/serenity-rs/songbird,
|
||||
README), used through `[patch.crates-io]` in `app/Cargo.toml` and `app/shell/Cargo.toml`.
|
||||
|
||||
The one change: RTP packets of an SSRC that Discord has not named yet are held for up to a second
|
||||
instead of dropped, and processed once its Speaking op names it (`src/driver/tasks/udp_rx/held.rs`,
|
||||
`udp_rx/mod.rs` `process_udp_message`; `ws.rs` sends the Speaking event before naming the SSRC so
|
||||
instead of dropped, and processed once its Speaking op names it: before the next packet, or on the
|
||||
next 20 ms playout tick when none follows (`src/driver/tasks/udp_rx/held.rs`, `udp_rx/mod.rs`
|
||||
`process_udp_message` and `release_named`; `ws.rs` sends the Speaking event before naming the SSRC so
|
||||
handlers see the name before the held packets). Discord's Speaking op comes after a speaker's first
|
||||
packets, and unpatched songbird dropped DAVE packets it could not yet decrypt, so the first ~100 ms
|
||||
of the first thing each person said after a join were lost (live 2026-11-18: "자비스 서울 날씨 어때"
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
//! and that op arrives after their first packets over UDP. Unpatched, songbird dropped DAVE packets
|
||||
//! of an unnamed SSRC (it needs the user to decrypt them), which cut the first ~100 ms off the first
|
||||
//! thing each person said after a join ("자비스" heard as "아비"). Held packets are processed in
|
||||
//! arrival order before the first packet after the name arrives.
|
||||
//! arrival order before the first packet after the name arrives, or on the next 20 ms playout tick
|
||||
//! when no packet follows (a short utterance that ended before its name came).
|
||||
|
||||
use bytes::BytesMut;
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
@@ -43,6 +44,11 @@ impl Held {
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// The SSRCs with held packets.
|
||||
pub(crate) fn ssrcs(&self) -> impl Iterator<Item = u32> + '_ {
|
||||
self.by_ssrc.keys().copied()
|
||||
}
|
||||
|
||||
/// Forgets every SSRC whose newest packet is older than `HOLD`.
|
||||
pub(crate) fn prune(&mut self, now: Instant) {
|
||||
self.by_ssrc.retain(|_, q| q.back().is_some_and(|(t, _)| now.duration_since(*t) <= HOLD));
|
||||
|
||||
@@ -87,6 +87,9 @@ impl UdpRx {
|
||||
}
|
||||
},
|
||||
() = tokio::time::sleep_until(playout_time) => {
|
||||
// Jarvis patch: a short first utterance may have no packet after its name came.
|
||||
self.release_named(interconnect).await;
|
||||
|
||||
let mut tick = VoiceTick {
|
||||
speaking: HashMap::new(),
|
||||
silent: HashSet::new(),
|
||||
@@ -171,6 +174,21 @@ impl UdpRx {
|
||||
self.process_packet(interconnect, packet).await;
|
||||
}
|
||||
|
||||
/// Jarvis patch: processes the held packets of every SSRC that has been named since (held.rs).
|
||||
async fn release_named(&mut self, interconnect: &Interconnect) {
|
||||
let named: Vec<u32> = self
|
||||
.held
|
||||
.ssrcs()
|
||||
.filter(|ssrc| self.ssrc_signalling.ssrc_user_map.contains_key(ssrc))
|
||||
.collect();
|
||||
let now = Instant::now();
|
||||
for ssrc in named {
|
||||
for held in self.held.take(ssrc, now) {
|
||||
self.process_packet(interconnect, held).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn process_packet(&mut self, interconnect: &Interconnect, mut packet: BytesMut) {
|
||||
// NOTE: errors here (and in general for UDP) are not fatal to the connection.
|
||||
// Panics should be avoided due to adversarial nature of rx'd packets,
|
||||
@@ -408,14 +426,9 @@ mod jarvis_held_tests {
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn packets_before_the_speaking_op_reach_handlers_once_it_names_the_ssrc() {
|
||||
let (core, _core_rx) = flume::unbounded();
|
||||
let (events, events_rx) = flume::unbounded();
|
||||
let (mixer, _mixer_rx) = flume::unbounded();
|
||||
let ic = Interconnect { core, events, mixer };
|
||||
async fn udp_rx() -> UdpRx {
|
||||
let mode = CryptoMode::Aes256Gcm;
|
||||
let mut rx = UdpRx {
|
||||
UdpRx {
|
||||
cipher: mode.cipher_from_key(&[0u8; 32]).unwrap(),
|
||||
crypto_mode: mode,
|
||||
decoder_map: HashMap::new(),
|
||||
@@ -426,7 +439,37 @@ mod jarvis_held_tests {
|
||||
dave_session: Arc::new(RwLock::new(None)),
|
||||
dave_protocol_version: Arc::new(AtomicU16::new(0)),
|
||||
held: Held::default(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_short_first_utterance_is_released_when_named_without_a_later_packet() {
|
||||
let (core, _core_rx) = flume::unbounded();
|
||||
let (events, events_rx) = flume::unbounded();
|
||||
let (mixer, _mixer_rx) = flume::unbounded();
|
||||
let ic = Interconnect { core, events, mixer };
|
||||
let mut rx = udp_rx().await;
|
||||
|
||||
rx.process_udp_message(&ic, rtp(1, 42)).await;
|
||||
rx.process_udp_message(&ic, rtp(2, 42)).await;
|
||||
rx.release_named(&ic).await;
|
||||
assert!(sequences(&events_rx).is_empty(), "still unnamed");
|
||||
|
||||
// The Speaking op comes after the last packet of the utterance.
|
||||
rx.ssrc_signalling.ssrc_user_map.insert(42, UserId(7));
|
||||
rx.release_named(&ic).await;
|
||||
assert_eq!(sequences(&events_rx), vec![1, 2]);
|
||||
rx.release_named(&ic).await;
|
||||
assert!(sequences(&events_rx).is_empty(), "released once");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn packets_before_the_speaking_op_reach_handlers_once_it_names_the_ssrc() {
|
||||
let (core, _core_rx) = flume::unbounded();
|
||||
let (events, events_rx) = flume::unbounded();
|
||||
let (mixer, _mixer_rx) = flume::unbounded();
|
||||
let ic = Interconnect { core, events, mixer };
|
||||
let mut rx = udp_rx().await;
|
||||
|
||||
rx.process_udp_message(&ic, rtp(1, 42)).await;
|
||||
rx.process_udp_message(&ic, rtp(2, 42)).await;
|
||||
|
||||
Reference in New Issue
Block a user