fix(voice): keep the first packets of a speaker until Discord names them

Discord names a speaker's SSRC in the Speaking op only the first time
they speak, and the op arrives after their first UDP packets. songbird
0.6.1 dropped the DAVE packets it could not decrypt yet, so the first
~100 ms of the first thing each person said after a join were lost
(live 2026-11-18: "자비스 서울 날씨 어때" arrived as "아비 서울날씨
어때", and the first call after the 01:20 start as "아비스").

songbird is vendored in app/vendor/songbird with one patch: packets of
an unnamed SSRC are held (1 s, 50 packets each, 32 SSRCs) and processed
in arrival order once it is named, and the Speaking event is sent before
the SSRC is named so handlers know the speaker first.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-18 16:15:07 +09:00
parent 88ee5ec2cb
commit 81d1e26f61
123 changed files with 20119 additions and 5 deletions

2
app/Cargo.lock generated
View File

@@ -3324,8 +3324,6 @@ dependencies = [
[[package]]
name = "songbird"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "371362adce10df3244fbf93475ac53391aedd425482f160a9dbb3e042577de8b"
dependencies = [
"aead",
"aes-gcm",

View File

@@ -44,6 +44,10 @@ jpeg-decoder = "0.3"
name = "vdisplay_elevated"
harness = false
# songbird 0.6.1 with the Jarvis patch for the first packets of a speaker (vendor/songbird/JARVIS-PATCH.md).
[patch.crates-io]
songbird = { path = "vendor/songbird" }
[profile.release]
lto = "thin"
codegen-units = 4

2
app/shell/Cargo.lock generated
View File

@@ -5562,8 +5562,6 @@ dependencies = [
[[package]]
name = "songbird"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "371362adce10df3244fbf93475ac53391aedd425482f160a9dbb3e042577de8b"
dependencies = [
"aead",
"aes-gcm",

View File

@@ -21,6 +21,10 @@ tauri-plugin-single-instance = "2"
serde_json = "1"
tokio = { version = "1", features = ["full"] }
# songbird 0.6.1 with the Jarvis patch for the first packets of a speaker (vendor/songbird/JARVIS-PATCH.md).
[patch.crates-io]
songbird = { path = "../vendor/songbird" }
[profile.release]
lto = "thin"
codegen-units = 4

View File

@@ -135,7 +135,14 @@ and the late "left" update cancelled the new join (seen on real Discord).
- **Hearing**: songbird only decrypts (transport and DAVE). Its own Opus decoding is not used: in
songbird 0.6.1 it decodes the payload without trimming what DAVE decryption left at its end, which
garbled, repeated or dropped about a third of the frames (measured with a test tone that changes
pitch every 20 ms). The core takes each decrypted RTP packet with songbird's exact payload bounds:
pitch every 20 ms). Discord names a speaker's SSRC (its Speaking op, over the websocket) only the
first time they speak, after their first packets arrive over UDP; songbird dropped the DAVE packets
it could not yet decrypt, which cut the first ~100 ms off the first thing each person said after a
join (live 2026-11-18: "자비스" heard as "아비"). The app therefore uses songbird 0.6.1 with a patch
(`app/vendor/songbird`, `JARVIS-PATCH.md`): packets of an SSRC not named yet are held, at most one
second and 50 packets for each of at most 32 SSRCs, and processed in arrival order once it is named;
the Speaking event goes out first, so the core knows the speaker before their held packets come.
The core takes each decrypted RTP packet with songbird's exact payload bounds:
- Packets go through a per-speaker reorder window in sequence-number order. A missing packet is
waited for until two later packets have arrived (about 40 ms), then rebuilt from the next packet's
Opus in-band FEC (the last missing frame) or concealed (any earlier ones). Late and duplicate

View File

@@ -0,0 +1,6 @@
{
"git": {
"sha1": "957e17fcbc687cd88979c90b76bdf79017803b09"
},
"path_in_vcs": ""
}

376
app/vendor/songbird/Cargo.toml vendored Normal file
View File

@@ -0,0 +1,376 @@
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies.
#
# If you are reading this file be aware that the original Cargo.toml
# will likely look very different (and much more reasonable).
# See Cargo.toml.orig for the original contents.
[package]
edition = "2021"
rust-version = "1.83.0"
name = "songbird"
version = "0.6.1"
authors = ["Kyle Simpson <kyleandrew.simpson@gmail.com>"]
build = "build.rs"
include = [
"src/**/*.rs",
"Cargo.toml",
"build.rs",
]
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "An async Rust library for the Discord voice API."
homepage = "https://github.com/serenity-rs/songbird"
documentation = "https://docs.rs/songbird"
readme = "README.md"
keywords = [
"discord",
"api",
"rtp",
"audio",
]
license = "ISC"
repository = "https://github.com/serenity-rs/songbird.git"
[package.metadata.docs.rs]
features = ["full-doc"]
rustdoc-args = [
"--cfg",
"docsrs",
]
[features]
builtin-queue = []
default = [
"driver",
"gateway",
"rustls",
"serenity",
"tungstenite",
]
driver = [
"dep:aead",
"dep:aes-gcm",
"dep:async-trait",
"dep:opus2",
"dep:byteorder",
"dep:bytes",
"dep:chacha20poly1305",
"dep:crypto-common",
"dep:davey",
"dep:discortp",
"dep:flume",
"dep:nohash-hasher",
"dep:parking_lot",
"dep:rand",
"dep:reqwest",
"dep:ringbuf",
"dep:rubato",
"dep:rusty_pool",
"dep:serde-aux",
"dep:serenity-voice-model",
"dep:socket2",
"dep:stream_lib",
"dep:streamcatcher",
"dep:symphonia",
"dep:symphonia-core",
"dep:tokio",
"dep:tokio-util",
"dep:typenum",
"dep:url",
"dep:uuid",
"tokio?/fs",
"tokio?/io-util",
"tokio?/macros",
"tokio?/net",
"tokio?/process",
"tokio?/rt",
"tokio?/sync",
"tokio?/time",
]
full-doc = [
"default",
"tungstenite",
"twilight",
"builtin-queue",
"receive",
]
gateway = [
"dep:async-trait",
"dep:dashmap",
"dep:flume",
"dep:parking_lot",
"dep:tokio",
"tokio?/sync",
"tokio?/time",
]
internals = ["dep:byteorder"]
native = [
"reqwest?/native-tls",
"serenity?/native_tls_backend",
"stream_lib?/native-tls",
"tokio-tungstenite?/native-tls",
"tokio-websockets?/native-tls",
"twilight-gateway?/native-tls",
]
receive = [
"dep:bytes",
"discortp?/demux",
"discortp?/rtcp",
]
rustls = [
"reqwest?/rustls-tls",
"serenity?/rustls_backend",
"stream_lib?/rustls-tls",
"tokio-tungstenite?/rustls-tls-webpki-roots",
"tokio-websockets?/ring",
"tokio-websockets?/rustls-native-roots",
"twilight-gateway?/rustls-native-roots",
]
tungstenite = ["dep:tokio-tungstenite"]
twilight = [
"dep:twilight-gateway",
"dep:twilight-model",
]
tws = ["dep:tokio-websockets"]
[lib]
name = "songbird"
path = "src/lib.rs"
bench = false
[dependencies.aead]
version = "0.5.2"
optional = true
[dependencies.aes-gcm]
version = "0.10.3"
optional = true
[dependencies.async-trait]
version = "0.1"
optional = true
[dependencies.byteorder]
version = "1"
optional = true
[dependencies.bytes]
version = "1"
optional = true
[dependencies.chacha20poly1305]
version = "0.10.1"
optional = true
[dependencies.crypto-common]
version = "0.1"
features = ["std"]
optional = true
[dependencies.dashmap]
version = "6.1.0"
optional = true
[dependencies.davey]
version = "0.1.2"
optional = true
[dependencies.derivative]
version = "2"
[dependencies.discortp]
version = "0.6"
features = [
"discord",
"pnet",
"rtp",
]
optional = true
default-features = false
[dependencies.either]
version = "1.9.0"
[dependencies.enum-map]
version = "2.7.3"
[dependencies.flume]
version = "0.11"
optional = true
[dependencies.futures]
version = "0.3"
[dependencies.nohash-hasher]
version = "0.2.0"
optional = true
[dependencies.nonmax]
version = "0.5.5"
[dependencies.opus2]
version = "0.4.0"
optional = true
[dependencies.parking_lot]
version = "0.12"
optional = true
[dependencies.pin-project]
version = "1"
[dependencies.rand]
version = "0.9"
optional = true
[dependencies.reqwest]
version = "0.12.2"
features = ["stream"]
optional = true
default-features = false
[dependencies.ringbuf]
version = "0.4"
optional = true
[dependencies.rubato]
version = "0.16"
optional = true
[dependencies.rusty_pool]
version = "0.7"
optional = true
[dependencies.serde]
version = "1"
features = ["derive"]
[dependencies.serde-aux]
version = "4"
optional = true
[dependencies.serde_json]
version = "1"
[dependencies.serenity]
version = "0.12.0"
features = [
"voice",
"gateway",
]
optional = true
default-features = false
[dependencies.serenity-voice-model]
version = "0.3"
optional = true
[dependencies.socket2]
version = "0.5"
optional = true
[dependencies.stream_lib]
version = "0.5.2"
optional = true
default-features = false
[dependencies.streamcatcher]
version = "1"
optional = true
[dependencies.symphonia]
version = "0.5.2"
optional = true
default-features = false
[dependencies.symphonia-core]
version = "0.5.2"
optional = true
[dependencies.tokio]
version = "1.0"
optional = true
default-features = false
[dependencies.tokio-tungstenite]
version = "0.26"
features = ["url"]
optional = true
[dependencies.tokio-util]
version = "0.7"
features = ["io"]
optional = true
[dependencies.tokio-websockets]
version = "0.11"
features = [
"client",
"fastrand",
"sha1_smol",
"simd",
]
optional = true
[dependencies.tracing]
version = "0.1"
features = ["log"]
[dependencies.tracing-futures]
version = "0.2"
[dependencies.twilight-gateway]
version = ">= 0.16.0, <= 0.17"
optional = true
default-features = false
[dependencies.twilight-model]
version = ">= 0.16.0, <= 0.17"
optional = true
default-features = false
[dependencies.typenum]
version = "1.17.0"
optional = true
[dependencies.url]
version = "2"
optional = true
[dependencies.uuid]
version = "1"
features = ["v4"]
optional = true
[dev-dependencies.byteorder]
version = "1"
[dev-dependencies.criterion]
version = "0.5"
[dev-dependencies.ntest]
version = "0.9"
[dev-dependencies.symphonia]
version = "0.5.2"
features = [
"aac",
"isomp4",
"mp3",
]
[dev-dependencies.tokio]
version = "1"
features = [
"rt",
"rt-multi-thread",
"test-util",
]

183
app/vendor/songbird/Cargo.toml.orig generated vendored Normal file
View File

@@ -0,0 +1,183 @@
[package]
authors = ["Kyle Simpson <kyleandrew.simpson@gmail.com>"]
description = "An async Rust library for the Discord voice API."
documentation = "https://docs.rs/songbird"
edition = "2021"
homepage = "https://github.com/serenity-rs/songbird"
include = ["src/**/*.rs", "Cargo.toml", "build.rs"]
keywords = ["discord", "api", "rtp", "audio"]
license = "ISC"
name = "songbird"
readme = "README.md"
repository = "https://github.com/serenity-rs/songbird.git"
rust-version = "1.83.0"
version = "0.6.1"
[dependencies]
aead = { optional = true, version = "0.5.2" }
aes-gcm = { optional = true, version = "0.10.3" }
async-trait = { optional = true, version = "0.1" }
opus2 = { optional = true, version = "0.4.0" }
byteorder = { optional = true, version = "1" }
bytes = { optional = true, version = "1" }
chacha20poly1305 = { optional = true, version = "0.10.1" }
crypto-common = { optional = true, features = ["std"], version = "0.1" }
dashmap = { optional = true, version = "6.1.0" }
davey = { optional = true, version = "0.1.2" }
derivative = "2"
discortp = { default-features = false, features = [
"discord",
"pnet",
"rtp",
], optional = true, version = "0.6" }
either = "1.9.0"
flume = { optional = true, version = "0.11" }
futures = "0.3"
nohash-hasher = { optional = true, version = "0.2.0" }
parking_lot = { optional = true, version = "0.12" }
pin-project = "1"
rand = { optional = true, version = "0.9" }
reqwest = { default-features = false, features = [
"stream",
], optional = true, version = "0.12.2" }
ringbuf = { optional = true, version = "0.4" }
rubato = { optional = true, version = "0.16" }
rusty_pool = { optional = true, version = "0.7" }
serde = { version = "1", features = ["derive"] }
serde-aux = { optional = true, version = "4" }
serde_json = "1"
serenity = { default-features = false, optional = true, version = "0.12.0", features = [
"voice",
"gateway",
] }
serenity-voice-model = { optional = true, version = "0.3" }
socket2 = { optional = true, version = "0.5" }
streamcatcher = { optional = true, version = "1" }
stream_lib = { default-features = false, optional = true, version = "0.5.2" }
symphonia = { default-features = false, optional = true, version = "0.5.2" }
symphonia-core = { optional = true, version = "0.5.2" }
tokio = { default-features = false, optional = true, version = "1.0" }
tokio-tungstenite = { optional = true, version = "0.26", features = ["url"] }
tokio-websockets = { optional = true, version = "0.11", features = [
"client",
"fastrand",
"sha1_smol",
"simd",
] }
tokio-util = { features = ["io"], optional = true, version = "0.7" }
tracing = { version = "0.1", features = ["log"] }
tracing-futures = "0.2"
twilight-gateway = { default-features = false, optional = true, version = ">= 0.16.0, <= 0.17" }
twilight-model = { default-features = false, optional = true, version = ">= 0.16.0, <= 0.17" }
typenum = { optional = true, version = "1.17.0" }
url = { optional = true, version = "2" }
uuid = { features = ["v4"], optional = true, version = "1" }
nonmax = "0.5.5"
enum-map = "2.7.3"
[dev-dependencies]
byteorder = "1"
criterion = "0.5"
ntest = "0.9"
symphonia = { version = "0.5.2", features = ["aac", "isomp4", "mp3"] }
tokio = { version = "1", features = ["rt", "rt-multi-thread", "test-util"] }
[features]
# Core features
default = ["driver", "gateway", "rustls", "serenity", "tungstenite"]
gateway = [
"dep:async-trait",
"dep:dashmap",
"dep:flume",
"dep:parking_lot",
"dep:tokio",
"tokio?/sync",
"tokio?/time",
]
driver = [
"dep:aead",
"dep:aes-gcm",
"dep:async-trait",
"dep:opus2",
"dep:byteorder",
"dep:bytes",
"dep:chacha20poly1305",
"dep:crypto-common",
"dep:davey",
"dep:discortp",
"dep:flume",
"dep:nohash-hasher",
"dep:parking_lot",
"dep:rand",
"dep:reqwest",
"dep:ringbuf",
"dep:rubato",
"dep:rusty_pool",
"dep:serde-aux",
"dep:serenity-voice-model",
"dep:socket2",
"dep:stream_lib",
"dep:streamcatcher",
"dep:symphonia",
"dep:symphonia-core",
"dep:tokio",
"dep:tokio-util",
"dep:typenum",
"dep:url",
"dep:uuid",
"tokio?/fs",
"tokio?/io-util",
"tokio?/macros",
"tokio?/net",
"tokio?/process",
"tokio?/rt",
"tokio?/sync",
"tokio?/time",
]
rustls = [
"reqwest?/rustls-tls",
"serenity?/rustls_backend",
"stream_lib?/rustls-tls",
"tokio-tungstenite?/rustls-tls-webpki-roots",
"tokio-websockets?/ring",
"tokio-websockets?/rustls-native-roots",
"twilight-gateway?/rustls-native-roots",
]
native = [
"reqwest?/native-tls",
"serenity?/native_tls_backend",
"stream_lib?/native-tls",
"tokio-tungstenite?/native-tls",
"tokio-websockets?/native-tls",
"twilight-gateway?/native-tls",
]
tungstenite = ["dep:tokio-tungstenite"]
tws = ["dep:tokio-websockets"]
twilight = ["dep:twilight-gateway", "dep:twilight-model"]
# Behaviour altering features.
builtin-queue = []
receive = ["dep:bytes", "discortp?/demux", "discortp?/rtcp"]
# Used for docgen/testing/benchmarking.
full-doc = ["default", "tungstenite", "twilight", "builtin-queue", "receive"]
internals = ["dep:byteorder"]
[lib]
bench = false
[[bench]]
name = "base-mixing"
path = "benches/base-mixing.rs"
required-features = ["internals"]
harness = false
[[bench]]
name = "mixing-task"
path = "benches/mixing-task.rs"
required-features = ["internals"]
harness = false
[package.metadata.docs.rs]
features = ["full-doc"]
rustdoc-args = ["--cfg", "docsrs"]

14
app/vendor/songbird/JARVIS-PATCH.md vendored Normal file
View File

@@ -0,0 +1,14 @@
# songbird 0.6.1, patched for Jarvis
This is songbird 0.6.1 from crates.io (https://github.com/serenity-rs/songbird, ISC licence, see its
README), used through `[patch.crates-io]` in `app/Cargo.toml` and `app/shell/Cargo.toml`.
The one change: RTP packets of an SSRC that Discord has not named yet are held for up to a second
instead of dropped, and processed once its Speaking op names it (`src/driver/tasks/udp_rx/held.rs`,
`udp_rx/mod.rs` `process_udp_message`; `ws.rs` sends the Speaking event before naming the SSRC so
handlers see the name before the held packets). Discord's Speaking op comes after a speaker's first
packets, and unpatched songbird dropped DAVE packets it could not yet decrypt, so the first ~100 ms
of the first thing each person said after a join were lost (live 2026-11-18: "자비스 서울 날씨 어때"
arrived as "아비 서울날씨 어때").
Drop this copy when an upstream release does the same.

97
app/vendor/songbird/README.md vendored Normal file
View File

@@ -0,0 +1,97 @@
[![crates.io badge]][crates.io] [![build badge]][build] [![docs badge]][docs] [![current docs badge]][current docs] [![next docs badge]][next docs] [![msrv badge]][msrv] [![license badge]][license] [![guild badge]][guild]
# Songbird
![](songbird.png)
Songbird is an async, cross-library compatible voice system for Discord, written in Rust.
The library offers:
* A standalone gateway frontend compatible with [serenity] and [twilight] using the
`"gateway"` and `"[serenity/twilight]"` plus `"[rustls/native]"` features. You can even run
driverless, to help manage your [lavalink] sessions.
* A standalone driver for voice calls, via the `"driver"` feature. If you can create
a `ConnectionInfo` using any other gateway, or language for your bot, then you
can run the songbird voice driver.
* Voice receive and RT(C)P packet handling via the `"receive"` feature.
* And, by default, a fully featured voice system featuring events, queues,
seeking on compatible streams, shared multithreaded audio stream caches,
and direct Opus data passthrough from DCA files.
## Intents
Songbird's gateway functionality requires you to specify the `GUILD_VOICE_STATES` intent.
## Codec support
Songbird supports all [codecs and formats provided by Symphonia] (pure-Rust), with Opus support
provided by [opus2] (an FFI wrapper for libopus).
**By default, *Songbird will not request any codecs from Symphonia*.** To change this, in your own
project you will need to depend on Symphonia as well.
```toml
# Including songbird alone gives you support for Opus via the DCA file format.
[dependencies.songbird]
version = "0.5"
features = ["builtin-queue"]
# To get additional codecs, you *must* add Symphonia yourself.
# This includes the default formats (MKV/WebM, Ogg, Wave) and codecs (FLAC, PCM, Vorbis)...
[dependencies.symphonia]
version = "0.5"
features = ["aac", "mp3", "isomp4", "alac"] # ...as well as any extras you need!
```
## Dependencies
Songbird needs a few system dependencies before you can use it.
- Opus - Audio codec that Discord uses.
[opus2] will use installed libopus binaries if available via pkgconf on Linux/MacOS, otherwise you will need to install cmake to build opus from source.
This is always the case on Windows.
For Unix systems, you can install the library with `apt install libopus-dev` on Ubuntu or `pacman -S opus` on Arch Linux.
If you do not have it installed it will be built for you. However, you will need a C compiler and the GNU autotools installed.
Again, these can be installed with `apt install build-essential autoconf automake libtool m4` on Ubuntu or `pacman -S base-devel` on Arch Linux.
This is a required dependency. Songbird cannot work without it.
- yt-dlp / youtube-dl / (similar forks) - Audio/Video download tool.
yt-dlp can be installed [according to the installation instructions on the main repo].
You can install youtube-dl with Python's package manager, pip, which we recommend for youtube-dl. You can do it with the command `pip install youtube_dl`.
Alternatively, you can install it with your system's package manager, `apt install youtube-dl` on Ubuntu or `pacman -S youtube-dl` on Arch Linux.
This is an optional dependency for users, but is required as a dev-dependency. It allows Songbird to download audio/video sources from the Internet from a variety of webpages, which it will convert to the Opus audio format Discord uses.
## Examples
Full examples showing various types of functionality and integrations can be found in [this crate's examples directory].
## Contributing
If you want to help out or file an issue, please look over [our contributor guidelines]!
## Attribution
Songbird's logo is based upon the copyright-free image ["Black-Capped Chickadee"] by George Gorgas White.
[serenity]: https://github.com/serenity-rs/serenity
[twilight]: https://github.com/twilight-rs/twilight
["Black-Capped Chickadee"]: https://www.oldbookillustrations.com/illustrations/black-capped-chickadee/
[lavalink]: https://github.com/freyacodes/Lavalink
[this crate's examples directory]: https://github.com/serenity-rs/songbird/tree/current/examples
[our contributor guidelines]: CONTRIBUTING.md
[codecs and formats provided by Symphonia]: https://github.com/pdeljanov/Symphonia#formats-demuxers
[opus2]: https://github.com/cijiugechu/opus2
[according to the installation instructions on the main repo]: https://github.com/yt-dlp/yt-dlp#installation
<!-- Badges -->
[crates.io]: https://crates.io/crates/songbird
[crates.io badge]: https://img.shields.io/crates/v/songbird.svg?style=flat-square
[build]: https://github.com/serenity-rs/songbird/actions
[build badge]: https://img.shields.io/github/actions/workflow/status/serenity-rs/songbird/ci.yml?branch=current&style=flat-square
[docs]: https://docs.rs/songbird/
[docs badge]: https://img.shields.io/badge/docs-online-informational?style=flat-square
[current docs]: https://serenity-rs.github.io/songbird/current/songbird/index.html
[current docs badge]: https://img.shields.io/badge/docs-current-4d76ae.svg?style=flat-square
[next docs]: https://serenity-rs.github.io/songbird/next/songbird/index.html
[next docs badge]: https://img.shields.io/badge/docs-next-4d76ae.svg?style=flat-square
[msrv]: https://blog.rust-lang.org/2024/11/28/Rust-1.83.0.html
[msrv badge]: https://img.shields.io/badge/rust-1.83+-93450a.svg?style=flat-square
[license]: LICENSE
[license badge]: https://img.shields.io/crates/l/songbird.svg?style=flat-square&color=yellow
[guild]: https://discord.gg/serenity-rs
[guild badge]: https://img.shields.io/discord/381880193251409931.svg?style=flat-square&colorB=7289DA

24
app/vendor/songbird/build.rs vendored Normal file
View File

@@ -0,0 +1,24 @@
#[cfg(all(feature = "driver", not(any(feature = "rustls", feature = "native"))))]
compile_error!(
"You have the `driver` feature enabled: \
either the `rustls` or `native` feature must be
selected to let Songbird's driver use websockets.\n\
- `rustls` uses Rustls, a pure Rust TLS-implemenation.\n\
- `native` uses SChannel on Windows, Secure Transport on macOS, \
and OpenSSL on other platforms.\n\
If you are unsure, go with `rustls`."
);
#[cfg(any(
all(feature = "driver", feature = "tws", feature = "tungstenite"),
all(feature = "driver", not(feature = "tws"), not(feature = "tungstenite"))
))]
compile_error!(
"You have the `driver` feature enabled: \
this requires you specify either: \n\
- `tungstenite` (recommended with serenity)\n\
- or `tws` (recommended with twilight).\n\
You have either specified none, or both - choose exactly one."
);
fn main() {}

444
app/vendor/songbird/src/config.rs vendored Normal file
View File

@@ -0,0 +1,444 @@
#[cfg(all(feature = "driver", feature = "receive"))]
use crate::driver::DecodeMode;
use crate::FloatDuration;
#[cfg(feature = "driver")]
use crate::{
driver::{
get_default_scheduler,
retry::Retry,
tasks::disposal::DisposalThread,
CryptoMode,
MixMode,
Scheduler,
},
input::codecs::*,
};
#[cfg(test)]
use crate::driver::test_config::*;
#[cfg(all(test, feature = "driver"))]
use crate::driver::SchedulerConfig;
#[cfg(feature = "driver")]
use symphonia::core::{codecs::CodecRegistry, probe::Probe};
use derivative::Derivative;
#[cfg(all(feature = "driver", feature = "receive"))]
use std::num::NonZeroU8;
use std::time::Duration;
/// Configuration for drivers and calls.
#[derive(Clone, Derivative)]
#[derivative(Debug)]
#[non_exhaustive]
pub struct Config {
#[cfg(feature = "driver")]
/// Selected tagging mode for voice packet encryption.
///
/// Defaults to [`CryptoMode::Aes256Gcm`].
///
/// Changes to this field will not immediately apply if the
/// driver is actively connected, but will apply to subsequent
/// sessions.
///
/// [`CryptoMode::Aes256Gcm`]: CryptoMode::Aes256Gcm
pub crypto_mode: CryptoMode,
#[cfg(all(feature = "driver", feature = "receive"))]
/// Configures whether decoding and decryption occur for all received packets.
///
/// If receiving and using voice packets, generally you should choose [`DecodeMode::Decode`].
/// [`DecodeMode::Decrypt`] is intended for users running their own selective decoding or
/// who need to inspect Opus packets. [User speaking state] can still be seen using [`DecodeMode::Pass`].
/// If you're certain you will never need any RT(C)P events, then consider building without
/// the `"receive"` feature for extra performance.
///
/// Defaults to [`DecodeMode::Decrypt`]. This is due to per-packet decoding costs,
/// which most users will not want to pay, but allowing speaking events which are commonly used.
///
/// [`DecodeMode::Decode`]: DecodeMode::Decode
/// [`DecodeMode::Decrypt`]: DecodeMode::Decrypt
/// [`DecodeMode::Pass`]: DecodeMode::Pass
/// [User speaking state]: crate::events::CoreEvent::VoiceTick
pub decode_mode: DecodeMode,
#[cfg(all(feature = "driver", feature = "receive"))]
/// Configures the amount of time after a user/SSRC is inactive before their decoder state
/// should be removed.
///
/// Defaults to 1 minute.
pub decode_state_timeout: FloatDuration,
#[cfg(all(feature = "driver", feature = "receive"))]
/// Configures the number of audio packets to buffer for each user before playout.
///
/// A playout buffer allows Songbird to smooth out jitter in audio packet arrivals,
/// as well as to correct for reordering of packets by the network.
///
/// This does not affect the arrival of raw packet events.
///
/// Defaults to 5 packets (100ms).
pub playout_buffer_length: NonZeroU8,
#[cfg(all(feature = "driver", feature = "receive"))]
/// Configures the initial amount of extra space allocated to handle packet bursts.
///
/// Each SSRC's receive buffer will start at capacity `playout_buffer_length +
/// playout_spike_length`, up to a maximum 64 packets.
///
/// Defaults to 3 packets (thus capacity defaults to 8).
pub playout_spike_length: u8,
#[cfg(feature = "gateway")]
/// Configures the amount of time to wait for Discord to reply with connection information
/// if [`Call::join`]/[`join_gateway`] are used.
///
/// This is a useful fallback in the event that:
/// * the underlying Discord client restarts and loses a join request, or
/// * a channel join fails because the bot is already believed to be there.
///
/// Defaults to 10 seconds. If set to `None`, connections will never time out.
///
/// [`Call::join`]: crate::Call::join
/// [`join_gateway`]: crate::Call::join_gateway
pub gateway_timeout: Option<FloatDuration>,
#[cfg(feature = "driver")]
/// Configures whether the driver will mix and output stereo or mono Opus data
/// over a voice channel.
///
/// Defaults to [`Stereo`].
///
/// [`Stereo`]: MixMode::Stereo
pub mix_mode: MixMode,
#[cfg(feature = "driver")]
/// Number of concurrently active tracks to allocate memory for.
///
/// This should be set at, or just above, the maximum number of tracks
/// you expect your bot will play at the same time. Exceeding the size of
/// the internal queue will trigger a larger memory allocation and copy,
/// possibly causing the mixer thread to miss a packet deadline.
///
/// Defaults to `1`.
///
/// Changes to this field in a running driver will only ever increase
/// the capacity of the track store.
pub preallocated_tracks: u8,
#[cfg(feature = "driver")]
/// Connection retry logic for the [`Driver`].
///
/// This controls how many times the [`Driver`] should retry any connections,
/// as well as how long to wait between attempts.
///
/// [`Driver`]: crate::driver::Driver
pub driver_retry: Retry,
#[cfg(feature = "driver")]
/// Configures whether or not each mixed audio packet is [soft-clipped] into the
/// [-1, 1] audio range.
///
/// Defaults to `true`, preventing clipping and dangerously loud audio from being sent.
///
/// **This operation adds ~3% cost to a standard (non-passthrough) mix cycle.**
/// If you *know* that your bot will only play one sound at a time and that
/// your volume is between `0.0` and `1.0`, then you can disable soft-clipping
/// for a performance boost. If you are playing several sounds at once, do not
/// disable this unless you make sure to reduce the volume of each sound.
///
/// [soft-clipped]: https://opus-codec.org/docs/opus_api-1.3.1/group__opus__decoder.html#gaff99598b352e8939dded08d96e125e0b
pub use_softclip: bool,
#[cfg(feature = "driver")]
/// Configures the maximum amount of time to wait for an attempted voice
/// connection to Discord.
///
/// Defaults to 10 seconds. If set to `None`, connections will never time out.
pub driver_timeout: Option<FloatDuration>,
#[cfg(feature = "driver")]
#[derivative(Debug = "ignore")]
/// Registry of the inner codecs supported by the driver, adding opus2-based
/// Opus codec support to all of Symphonia's default codecs.
///
/// Defaults to [`get_codec_registry`].
pub codec_registry: &'static CodecRegistry,
#[cfg(feature = "driver")]
#[derivative(Debug = "ignore")]
/// Registry of the muxers and container formats supported by the driver.
///
/// Defaults to [`get_probe`], which includes all of Symphonia's default format handlers
/// and DCA format support.
pub format_registry: &'static Probe,
#[cfg(feature = "driver")]
/// The Sender for a channel that will run the destructor of possibly blocking values.
///
/// If not set, a thread will be spawned to perform this, but it is recommended to create
/// a long running thread instead of relying on a per-driver thread.
///
/// Note: When using [`Songbird`] this is overwritten automatically by its disposal thread.
///
/// [`Songbird`]: crate::Songbird
pub disposer: Option<DisposalThread>,
#[cfg(feature = "driver")]
/// The scheduler is responsible for mapping idle and active [`Driver`] instances
/// to threads.
///
/// If set to None, then songbird will use [`get_default_scheduler`].
///
/// [`Driver`]: crate::Driver
pub scheduler: Option<Scheduler>,
// Test only attributes
#[cfg(feature = "driver")]
#[cfg(test)]
/// Test config to offer precise control over mixing tick rate/count.
pub(crate) tick_style: TickStyle,
#[cfg(feature = "driver")]
#[cfg(test)]
/// If set, skip connection and encryption steps.
pub(crate) override_connection: Option<OutputMode>,
}
impl Default for Config {
fn default() -> Self {
Self {
#[cfg(feature = "driver")]
crypto_mode: CryptoMode::Aes256Gcm,
#[cfg(all(feature = "driver", feature = "receive"))]
decode_mode: DecodeMode::Decrypt,
#[cfg(all(feature = "driver", feature = "receive"))]
decode_state_timeout: Duration::from_secs(60).into(),
#[cfg(all(feature = "driver", feature = "receive"))]
playout_buffer_length: NonZeroU8::new(5).unwrap(),
#[cfg(all(feature = "driver", feature = "receive"))]
playout_spike_length: 3,
#[cfg(feature = "gateway")]
gateway_timeout: Some(Duration::from_secs(10).into()),
#[cfg(feature = "driver")]
mix_mode: MixMode::Stereo,
#[cfg(feature = "driver")]
preallocated_tracks: 1,
#[cfg(feature = "driver")]
use_softclip: true,
#[cfg(feature = "driver")]
driver_retry: Retry::default(),
#[cfg(feature = "driver")]
driver_timeout: Some(Duration::from_secs(10).into()),
#[cfg(feature = "driver")]
codec_registry: get_codec_registry(),
#[cfg(feature = "driver")]
format_registry: get_probe(),
#[cfg(feature = "driver")]
disposer: None,
#[cfg(feature = "driver")]
scheduler: None,
#[cfg(feature = "driver")]
#[cfg(test)]
tick_style: TickStyle::Timed,
#[cfg(feature = "driver")]
#[cfg(test)]
override_connection: None,
}
}
}
#[cfg(feature = "driver")]
impl Config {
/// Sets this `Config`'s chosen cryptographic tagging scheme.
#[must_use]
pub fn crypto_mode(mut self, crypto_mode: CryptoMode) -> Self {
self.crypto_mode = crypto_mode;
self
}
#[cfg(feature = "receive")]
/// Sets this `Config`'s received packet decryption/decoding behaviour.
#[must_use]
pub fn decode_mode(mut self, decode_mode: DecodeMode) -> Self {
self.decode_mode = decode_mode;
self
}
#[cfg(feature = "receive")]
/// Sets this `Config`'s received packet decoder cleanup timer.
#[must_use]
pub fn decode_state_timeout(mut self, decode_state_timeout: Duration) -> Self {
self.decode_state_timeout = decode_state_timeout.into();
self
}
#[cfg(feature = "receive")]
/// Sets this `Config`'s playout buffer length, in packets.
#[must_use]
pub fn playout_buffer_length(mut self, playout_buffer_length: NonZeroU8) -> Self {
self.playout_buffer_length = playout_buffer_length;
self
}
#[cfg(feature = "receive")]
/// Sets this `Config`'s additional pre-allocated space to handle bursty audio packets.
#[must_use]
pub fn playout_spike_length(mut self, playout_spike_length: u8) -> Self {
self.playout_spike_length = playout_spike_length;
self
}
/// Sets this `Config`'s audio mixing channel count.
#[must_use]
pub fn mix_mode(mut self, mix_mode: MixMode) -> Self {
self.mix_mode = mix_mode;
self
}
/// Sets this `Config`'s number of tracks to preallocate.
#[must_use]
pub fn preallocated_tracks(mut self, preallocated_tracks: u8) -> Self {
self.preallocated_tracks = preallocated_tracks;
self
}
/// Sets this `Config`'s number to enable/disable soft-clipping sent audio.
#[must_use]
pub fn use_softclip(mut self, use_softclip: bool) -> Self {
self.use_softclip = use_softclip;
self
}
/// Sets this `Config`'s timeout for establishing a voice connection.
#[must_use]
pub fn driver_timeout(mut self, driver_timeout: Option<Duration>) -> Self {
self.driver_timeout = driver_timeout.map(Into::into);
self
}
/// Sets this `Config`'s voice connection retry configuration.
#[must_use]
pub fn driver_retry(mut self, driver_retry: Retry) -> Self {
self.driver_retry = driver_retry;
self
}
/// Sets this `Config`'s symphonia codec registry.
#[must_use]
pub fn codec_registry(mut self, codec_registry: &'static CodecRegistry) -> Self {
self.codec_registry = codec_registry;
self
}
/// Sets this `Config`'s symphonia format registry/probe set.
#[must_use]
pub fn format_registry(mut self, format_registry: &'static Probe) -> Self {
self.format_registry = format_registry;
self
}
/// Sets this `Config`'s channel for sending disposal messages.
#[must_use]
pub fn disposer(mut self, disposer: DisposalThread) -> Self {
self.disposer = Some(disposer);
self
}
/// Sets this `Config`'s mixer scheduler.
#[must_use]
pub fn scheduler(mut self, scheduler: Scheduler) -> Self {
self.scheduler = Some(scheduler);
self
}
/// Returns a lightweight reference to the audio scheduler this `Config` will use.
#[must_use]
pub fn get_scheduler(&self) -> Scheduler {
self.scheduler
.as_ref()
.unwrap_or(get_default_scheduler())
.clone()
}
/// Ensures a global disposer has been set, initializing one if not.
#[must_use]
pub(crate) fn initialise_disposer(self) -> Self {
if self.disposer.is_some() {
self
} else {
self.disposer(DisposalThread::run())
}
}
/// This is used to prevent changes which would invalidate the current session.
pub(crate) fn make_safe(&mut self, previous: &Config, connected: bool) {
if connected {
self.crypto_mode = previous.crypto_mode;
}
}
}
#[cfg(not(feature = "driver"))]
impl Config {
pub(crate) fn initialise_disposer(self) -> Self {
self
}
}
// Test only attributes
#[cfg(all(test, feature = "driver"))]
#[allow(missing_docs)]
impl Config {
#[must_use]
pub fn tick_style(mut self, tick_style: TickStyle) -> Self {
self.tick_style = tick_style;
self
}
/// Sets this `Config`'s voice connection retry configuration.
#[must_use]
pub fn override_connection(mut self, override_connection: Option<OutputMode>) -> Self {
self.override_connection = override_connection;
self
}
#[must_use]
pub fn test_cfg(raw_output: bool) -> (DriverTestHandle, Config) {
let (tick_tx, tick_rx) = flume::unbounded();
let (conn, rx) = if raw_output {
let (pkt_tx, pkt_rx) = flume::unbounded();
(OutputMode::Raw(pkt_tx), OutputReceiver::Raw(pkt_rx))
} else {
let (rtp_tx, rtp_rx) = flume::unbounded();
(OutputMode::Rtp(rtp_tx), OutputReceiver::Rtp(rtp_rx))
};
let sc_config = SchedulerConfig {
strategy: crate::driver::SchedulerMode::MaxPerThread(1.try_into().unwrap()),
move_expensive_tasks: true,
};
let config = Config::default()
.tick_style(TickStyle::UntimedWithExecLimit(tick_rx))
// give each test its own thread in the scheduler for simplicity.
.scheduler(Scheduler::new(sc_config))
.override_connection(Some(conn));
let handle = DriverTestHandle { rx, tx: tick_tx };
(handle, config)
}
}
#[cfg(feature = "gateway")]
impl Config {
/// Sets this `Config`'s timeout for joining a voice channel.
#[must_use]
pub fn gateway_timeout(mut self, gateway_timeout: Option<Duration>) -> Self {
self.gateway_timeout = gateway_timeout.map(Into::into);
self
}
}

136
app/vendor/songbird/src/constants.rs vendored Normal file
View File

@@ -0,0 +1,136 @@
//! Constants affecting driver function and API handling.
#[cfg(feature = "driver")]
use discortp::rtp::RtpType;
#[cfg(feature = "driver")]
use opus2::Bitrate;
use std::time::Duration;
#[cfg(feature = "driver")]
/// The voice gateway version used by the library.
pub const VOICE_GATEWAY_VERSION: u8 = crate::model::constants::GATEWAY_VERSION;
#[cfg(feature = "driver")]
/// Sample rate of audio to be sent to Discord.
pub const SAMPLE_RATE: u32 = 48000;
/// Sample rate of audio to be sent to Discord.
pub const SAMPLE_RATE_RAW: usize = 48_000;
/// Number of audio frames/packets to be sent per second.
pub const AUDIO_FRAME_RATE: usize = 50;
/// Length of time between any two audio frames.
pub const TIMESTEP_LENGTH: Duration = Duration::from_millis(1000 / AUDIO_FRAME_RATE as u64);
#[cfg(feature = "driver")]
/// Default bitrate for audio.
pub const DEFAULT_BITRATE: Bitrate = Bitrate::Bits(128_000);
/// Number of output samples at 48kHZ to produced when resampling subframes.
pub(crate) const RESAMPLE_OUTPUT_FRAME_SIZE: usize = MONO_FRAME_SIZE / 2;
/// The maximum number of bad frames to allow in an Opus source before blocking passthrough.
pub(crate) const OPUS_PASSTHROUGH_STRIKE_LIMIT: u8 = 3;
/// Number of samples in one complete frame of audio per channel.
///
/// This is equally the number of stereo (joint) samples in an audio frame.
pub const MONO_FRAME_SIZE: usize = SAMPLE_RATE_RAW / AUDIO_FRAME_RATE;
/// Number of individual samples in one complete frame of stereo audio.
pub const STEREO_FRAME_SIZE: usize = 2 * MONO_FRAME_SIZE;
/// Number of bytes in one complete frame of raw `f32`-encoded mono audio.
pub const MONO_FRAME_BYTE_SIZE: usize = MONO_FRAME_SIZE * std::mem::size_of::<f32>();
/// Number of bytes in one complete frame of raw `f32`-encoded stereo audio.
pub const STEREO_FRAME_BYTE_SIZE: usize = STEREO_FRAME_SIZE * std::mem::size_of::<f32>();
/// Length (in milliseconds) of any audio frame.
pub const FRAME_LEN_MS: usize = 1000 / AUDIO_FRAME_RATE;
/// Maximum number of audio frames/packets to be sent per second to be buffered.
pub const CHILD_BUFFER_LEN: usize = AUDIO_FRAME_RATE / 2;
/// Maximum packet size for a voice packet.
///
/// Set a safe amount below the Ethernet MTU to avoid fragmentation/rejection.
pub const VOICE_PACKET_MAX: usize = 1460;
/// Delay between sends of UDP keepalive frames.
///
/// Passive monitoring of Discord itself shows that these fire every 5 seconds
/// irrespective of outgoing UDP traffic.
pub const UDP_KEEPALIVE_GAP_MS: u64 = 5_000;
/// Type-converted delay between sends of UDP keepalive frames.
///
/// Passive monitoring of Discord itself shows that these fire every 5 seconds
/// irrespective of outgoing UDP traffic.
pub const UDP_KEEPALIVE_GAP: Duration = Duration::from_millis(UDP_KEEPALIVE_GAP_MS);
/// Opus silent frame, used to signal speech start and end (and prevent audio glitching).
pub const SILENT_FRAME: [u8; 3] = [0xf8, 0xff, 0xfe];
/// The one (and only) RTP version.
pub const RTP_VERSION: u8 = 2;
#[cfg(feature = "driver")]
/// Profile type used by Discord's Opus audio traffic.
pub const RTP_PROFILE_TYPE: RtpType = RtpType::Dynamic(120);
#[cfg(feature = "receive")]
/// The magic marker appended to the end of DAVE encrypted payloads, used to determine
/// whether the voice packet is encrypted.
pub const DAVE_MAGIC_MARKER: [u8; 2] = [0xfa, 0xfa];
#[cfg(test)]
#[allow(clippy::doc_markdown, missing_docs)]
pub mod test_data {
/// URL for a source which YTDL must extract.
///
/// Referenced under CC BY-NC-SA 3.0 -- https://creativecommons.org/licenses/by-nc-sa/3.0/
pub const YTDL_TARGET: &str = "https://cloudkicker.bandcamp.com/track/94-days";
/// URL for a source that has both a playlist and a music video,
/// which YTDL should extract.
///
/// Referenced under CC BY-NC-SA 3.0 -- https://creativecommons.org/licenses/by-nc-sa/3.0/
pub const YTDL_PLAYLIST_TARGET: &str =
"https://www.youtube.com/watch?v=KSgEFfWZ-W0&list=OLAK5uy_l2x81ffbpevMSjUn7NniL_rNLulWM3n6g&index=7";
/// URL for a source which can be read via an Http Request.
///
/// Referenced under CC BY-NC-SA 3.0 -- https://creativecommons.org/licenses/by-nc-sa/3.0/
pub const HTTP_TARGET: &str = "https://github.com/FelixMcFelix/songbird/raw/symphonia/resources/Cloudkicker%20-%202011%2007.mp3";
/// URL for an opus/ogg source which can be read via an Http Request.
///
/// Referenced under CC BY 3.0 -- https://creativecommons.org/licenses/by/3.0/
pub const HTTP_OPUS_TARGET: &str = "https://github.com/FelixMcFelix/songbird/raw/symphonia/resources/Cloudkicker%20-%20Making%20Will%20Mad.opus";
/// URL for an opus/webm source which can be read via an Http Request.
///
/// Referenced under CC BY 3.0 -- https://creativecommons.org/licenses/by/3.0/
pub const HTTP_WEBM_TARGET: &str = "https://github.com/FelixMcFelix/songbird/raw/symphonia/resources/Cloudkicker%20-%20Making%20Will%20Mad.webm";
/// Path to a DCA source.
///
/// Referenced under CC BY-NC-SA 3.0 -- https://creativecommons.org/licenses/by-nc-sa/3.0/
pub const FILE_DCA_TARGET: &str = "resources/Cloudkicker - 2011 07.dca1";
/// Path to an opus source which can be read via a File.
///
/// Referenced under CC BY 3.0 -- https://creativecommons.org/licenses/by/3.0/
pub const FILE_WEBM_TARGET: &str = "resources/Cloudkicker - Making Will Mad.webm";
/// Path to a Wav source which can be read via a File.
pub const FILE_WAV_TARGET: &str = "resources/loop.wav";
/// Path to a shorter MP3 source which can be read via a File.
pub const FILE_SHORT_MP3_TARGET: &str = "resources/ting.mp3";
/// Path to an MP4 (H264 + AAC) source which can be read via a File.
pub const FILE_VID_TARGET: &str = "resources/ting-vid.mp4";
}

View File

@@ -0,0 +1,24 @@
//! Various driver internals which need to be exported for benchmarking.
//!
//! Included if using the `"internals"` feature flag.
//! You should not and/or cannot use these as part of a normal application.
#![allow(missing_docs)]
pub use super::tasks::{message as task_message, mixer};
pub use super::crypto::{Cipher, CryptoState};
use crate::{
driver::tasks::message::TrackContext,
tracks::{Track, TrackHandle},
};
#[must_use]
pub fn track_context(t: Track) -> (TrackHandle, TrackContext) {
t.into_context()
}
pub mod scheduler {
pub use crate::driver::scheduler::*;
}

View File

@@ -0,0 +1,152 @@
//! Connection errors and convenience types.
use crate::{
driver::tasks::{error::Recipient, message::*},
ws::Error as WsError,
};
use aes_gcm::Error as CryptoError;
use davey::errors::{
CreateKeyPackageError as DaveyCreateKeyPackageError,
InitError as DaveyInitError,
};
use flume::SendError;
use serde_json::Error as JsonError;
use std::{error::Error as StdError, fmt, io::Error as IoError};
use tokio::time::error::Elapsed;
/// Errors encountered while connecting to a Discord voice server over the driver.
#[derive(Debug)]
#[non_exhaustive]
pub enum Error {
/// The driver hung up an internal signaller, either due to another connection attempt
/// or a crash.
AttemptDiscarded,
/// An error occurred during [en/de]cryption of voice packets.
Crypto(CryptoError),
/// The symmetric key supplied by Discord had the wrong size.
CryptoInvalidLength,
/// Server did not return the expected crypto mode during negotiation.
CryptoModeInvalid,
/// Selected crypto mode was not offered by server.
CryptoModeUnavailable,
/// Failed to create a DAVE key package.
#[cfg(feature = "driver")]
DaveCreateKeyPackageError(DaveyCreateKeyPackageError),
/// An error occured during initialization of the DAVE session.
#[cfg(feature = "driver")]
DaveInitializationError(DaveyInitError),
/// An indicator that an endpoint URL was invalid.
EndpointUrl,
/// Discord failed to correctly respond to IP discovery.
IllegalDiscoveryResponse,
/// Could not parse Discord's view of our IP.
IllegalIp,
/// Miscellaneous I/O error.
Io(IoError),
/// JSON (de)serialization error.
Json(JsonError),
/// Failed to message other background tasks after connection establishment.
InterconnectFailure(Recipient),
/// Error communicating with gateway server over WebSocket.
Ws(WsError),
/// Connection attempt timed out.
TimedOut,
}
impl From<CryptoError> for Error {
fn from(e: CryptoError) -> Self {
Error::Crypto(e)
}
}
impl From<IoError> for Error {
fn from(e: IoError) -> Error {
Error::Io(e)
}
}
impl From<JsonError> for Error {
fn from(e: JsonError) -> Error {
Error::Json(e)
}
}
impl From<SendError<WsMessage>> for Error {
fn from(_e: SendError<WsMessage>) -> Error {
Error::InterconnectFailure(Recipient::AuxNetwork)
}
}
impl From<SendError<EventMessage>> for Error {
fn from(_e: SendError<EventMessage>) -> Error {
Error::InterconnectFailure(Recipient::Event)
}
}
impl From<SendError<MixerMessage>> for Error {
fn from(_e: SendError<MixerMessage>) -> Error {
Error::InterconnectFailure(Recipient::Mixer)
}
}
impl From<WsError> for Error {
fn from(e: WsError) -> Error {
Error::Ws(e)
}
}
impl From<Elapsed> for Error {
fn from(_e: Elapsed) -> Error {
Error::TimedOut
}
}
impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "failed to connect to Discord RTP server: ")?;
match self {
Self::AttemptDiscarded => write!(f, "connection attempt was aborted/discarded"),
Self::Crypto(e) => e.fmt(f),
Self::CryptoInvalidLength => write!(f, "server supplied key of wrong length"),
Self::CryptoModeInvalid => write!(f, "server changed negotiated encryption mode"),
Self::CryptoModeUnavailable => write!(f, "server did not offer chosen encryption mode"),
Self::DaveCreateKeyPackageError(e) => e.fmt(f),
Self::DaveInitializationError(e) => e.fmt(f),
Self::EndpointUrl => write!(f, "endpoint URL received from gateway was invalid"),
Self::IllegalDiscoveryResponse => {
write!(f, "IP discovery/NAT punching response was invalid")
},
Self::IllegalIp => write!(f, "IP discovery/NAT punching response had bad IP value"),
Self::Io(e) => e.fmt(f),
Self::Json(e) => e.fmt(f),
Self::InterconnectFailure(e) => write!(f, "failed to contact other task ({e:?})"),
Self::Ws(e) => write!(f, "websocket issue ({e:?})."),
Self::TimedOut => write!(f, "connection attempt timed out"),
}
}
}
impl StdError for Error {
fn source(&self) -> Option<&(dyn StdError + 'static)> {
match self {
Error::AttemptDiscarded
| Error::Crypto(_)
| Error::CryptoInvalidLength
| Error::CryptoModeInvalid
| Error::CryptoModeUnavailable
| Error::DaveCreateKeyPackageError(_)
| Error::DaveInitializationError(_)
| Error::EndpointUrl
| Error::IllegalDiscoveryResponse
| Error::IllegalIp
| Error::InterconnectFailure(_)
| Error::Ws(_)
| Error::TimedOut => None,
Error::Io(e) => e.source(),
Error::Json(e) => e.source(),
}
}
}
/// Convenience type for Discord voice/driver connection error handling.
pub type Result<T> = std::result::Result<T, Error>;

View File

@@ -0,0 +1,413 @@
pub mod error;
#[cfg(feature = "receive")]
use super::tasks::udp_rx;
use super::{
crypto::Cipher,
tasks::{
message::*,
ws::{self as ws_task, AuxNetwork},
},
Config,
CryptoMode,
};
use crate::{
constants::*,
model::{
payload::{Identify, Resume, SelectProtocol},
Event as GatewayEvent,
ProtocolData,
},
ws::WsStream,
ConnectionInfo,
};
use discortp::discord::{IpDiscoveryPacket, IpDiscoveryType, MutableIpDiscoveryPacket};
use error::{Error, Result};
use flume::Sender;
use serenity_voice_model::payload::DaveMlsKeyPackage;
use socket2::Socket;
use std::sync::{atomic::AtomicU16, Arc, RwLock};
use std::{net::IpAddr, num::NonZeroU16, str::FromStr};
use tokio::{net::UdpSocket, spawn, time::timeout};
use tracing::{debug, info, instrument};
use url::Url;
pub(crate) struct Connection {
pub(crate) info: ConnectionInfo,
pub(crate) ssrc: u32,
pub(crate) ws: Sender<WsMessage>,
}
impl Connection {
pub(crate) async fn new(
info: ConnectionInfo,
interconnect: &Interconnect,
config: &Config,
idx: usize,
) -> Result<Connection> {
if let Some(t) = config.driver_timeout {
timeout(
t.into(),
Connection::new_inner(info, interconnect, config, idx),
)
.await?
} else {
Connection::new_inner(info, interconnect, config, idx).await
}
}
pub(crate) async fn new_inner(
info: ConnectionInfo,
interconnect: &Interconnect,
config: &Config,
idx: usize,
) -> Result<Connection> {
let url = generate_url(&info.endpoint)?;
let mut client = WsStream::connect(url).await?;
let (ws_msg_tx, ws_msg_rx) = flume::unbounded();
let mut hello = None;
let mut ready = None;
client
.send_json(&GatewayEvent::from(Identify {
server_id: info.guild_id.into(),
session_id: info.session_id.clone(),
token: info.token.clone(),
user_id: info.user_id.into(),
max_dave_protocol_version: Some(davey::DAVE_PROTOCOL_VERSION),
}))
.await?;
loop {
let Some(value) = client.recv_event().await? else {
continue;
};
match value {
GatewayEvent::Ready(r) => {
ready = Some(r);
if hello.is_some() {
break;
}
},
GatewayEvent::Hello(h) => {
hello = Some(h);
if ready.is_some() {
break;
}
},
other => {
// Discord hold back per-user connection state until after this handshake.
// There's no guarantee that will remain the case, so buffer it like all
// subsequent steps where we know they *do* send these packets.
debug!("Expected ready/hello; got: {:?}", other);
ws_msg_tx.send(WsMessage::Deliver(other))?;
},
}
}
let hello =
hello.expect("Hello packet expected in connection initialisation, but not found.");
let ready =
ready.expect("Ready packet expected in connection initialisation, but not found.");
let chosen_crypto = CryptoMode::negotiate(&ready.modes, Some(config.crypto_mode))?;
info!(
"Crypto scheme negotiation -- wanted {:?}. Chose {:?} from modes {:?}.",
config.crypto_mode, chosen_crypto, ready.modes
);
let udp = UdpSocket::bind("0.0.0.0:0").await?;
// Optimisation for non-receive case: set rx buffer size to zero.
let udp = if cfg!(feature = "receive") {
udp
} else {
let socket = Socket::from(udp.into_std()?);
// Some operating systems do not allow setting the recv buffer to 0.
#[cfg(any(target_os = "linux", target_os = "windows"))]
socket.set_recv_buffer_size(0)?;
UdpSocket::from_std(socket.into())?
};
udp.connect((ready.ip, ready.port)).await?;
// Follow Discord's IP Discovery procedures, in case NAT tunnelling is needed.
let mut bytes = [0; IpDiscoveryPacket::const_packet_size()];
{
let mut view = MutableIpDiscoveryPacket::new(&mut bytes[..]).expect(
"Too few bytes in 'bytes' for IPDiscovery packet.\
(Blame: IpDiscoveryPacket::const_packet_size()?)",
);
view.set_pkt_type(IpDiscoveryType::Request);
view.set_length(70);
view.set_ssrc(ready.ssrc);
}
udp.send(&bytes).await?;
let (len, _addr) = udp.recv_from(&mut bytes).await?;
{
let view =
IpDiscoveryPacket::new(&bytes[..len]).ok_or(Error::IllegalDiscoveryResponse)?;
if view.get_pkt_type() != IpDiscoveryType::Response {
return Err(Error::IllegalDiscoveryResponse);
}
// We could do something clever like binary search,
// but possibility of UDP spoofing precludes us from
// making the assumption we can find a "left edge" of '\0's.
let nul_byte_index = view
.get_address_raw()
.iter()
.position(|&b| b == 0)
.ok_or(Error::IllegalIp)?;
let address_str = std::str::from_utf8(&view.get_address_raw()[..nul_byte_index])
.map_err(|_| Error::IllegalIp)?;
let address = IpAddr::from_str(address_str).map_err(|_| Error::IllegalIp)?;
client
.send_json(&GatewayEvent::from(SelectProtocol {
protocol: "udp".into(),
data: ProtocolData {
address,
mode: chosen_crypto.to_request_str().into(),
port: view.get_port(),
},
}))
.await?;
}
let (cipher, dave_session, dave_protocol_version) =
init_cipher(&mut client, &info, chosen_crypto, &ws_msg_tx).await?;
let dave_session = Arc::new(RwLock::new(dave_session));
let dave_protocol_version = Arc::new(dave_protocol_version);
info!("Connected to: {}", info.endpoint);
info!("WS heartbeat duration {}ms.", hello.heartbeat_interval);
#[cfg(feature = "receive")]
let (udp_receiver_msg_tx, udp_receiver_msg_rx) = flume::unbounded();
// NOTE: This causes the UDP Socket on "receive" to be non-blocking,
// and the standard to be blocking. A UDP send should only WouldBlock if
// you're sending more data than the OS can handle (not likely, and
// at that point you should scale horizontally).
//
// If this is a problem for anyone, we can make non-blocking sends
// queue up a delayed send up to a limit.
#[cfg(feature = "receive")]
let (udp_rx, udp_tx) = {
let udp_tx = udp.into_std()?;
let udp_rx = UdpSocket::from_std(udp_tx.try_clone()?)?;
(udp_rx, udp_tx)
};
#[cfg(not(feature = "receive"))]
let udp_tx = udp.into_std()?;
let ssrc = ready.ssrc;
let mix_conn = MixerConnection {
#[cfg(feature = "receive")]
cipher: cipher.clone(),
#[cfg(not(feature = "receive"))]
cipher,
dave_session: dave_session.clone(),
dave_protocol_version: dave_protocol_version.clone(),
crypto_state: chosen_crypto.into(),
#[cfg(feature = "receive")]
udp_rx: udp_receiver_msg_tx,
udp_tx,
};
interconnect
.mixer
.send(MixerMessage::Ws(Some(ws_msg_tx.clone())))?;
interconnect
.mixer
.send(MixerMessage::SetConn(mix_conn, ready.ssrc))?;
#[cfg(feature = "receive")]
let ssrc_tracker = Arc::new(SsrcTracker::default());
let ws_state = AuxNetwork::new(
ws_msg_rx,
client,
ssrc,
hello.heartbeat_interval,
idx,
info.clone(),
#[cfg(not(feature = "receive"))]
dave_session,
#[cfg(not(feature = "receive"))]
dave_protocol_version,
#[cfg(feature = "receive")]
dave_session.clone(),
#[cfg(feature = "receive")]
dave_protocol_version.clone(),
#[cfg(feature = "receive")]
ssrc_tracker.clone(),
);
spawn(ws_task::runner(interconnect.clone(), ws_state));
#[cfg(feature = "receive")]
spawn(udp_rx::runner(
interconnect.clone(),
udp_receiver_msg_rx,
cipher,
chosen_crypto,
config.clone(),
udp_rx,
ssrc_tracker,
dave_session,
dave_protocol_version,
));
Ok(Connection {
info,
ssrc,
ws: ws_msg_tx,
})
}
#[instrument(skip(self))]
pub async fn reconnect(&mut self, config: &Config) -> Result<()> {
if let Some(t) = config.driver_timeout {
timeout(t.into(), self.reconnect_inner()).await?
} else {
self.reconnect_inner().await
}
}
#[instrument(skip(self))]
pub async fn reconnect_inner(&mut self) -> Result<()> {
let url = generate_url(&self.info.endpoint)?;
// Thread may have died, we want to send to prompt a clean exit
// (if at all possible) and then proceed as normal.
let mut client = WsStream::connect(url).await?;
client
.send_json(&GatewayEvent::from(Resume {
server_id: self.info.guild_id.into(),
session_id: self.info.session_id.clone(),
token: self.info.token.clone(),
}))
.await?;
let mut hello = None;
let mut resumed = None;
loop {
let Some(value) = client.recv_event().await? else {
continue;
};
match value {
GatewayEvent::Resumed => {
resumed = Some(());
if hello.is_some() {
break;
}
},
GatewayEvent::Hello(h) => {
hello = Some(h);
if resumed.is_some() {
break;
}
},
other => {
self.ws.send(WsMessage::Deliver(other))?;
},
}
}
let hello =
hello.expect("Hello packet expected in connection initialisation, but not found.");
self.ws
.send(WsMessage::SetKeepalive(hello.heartbeat_interval))?;
self.ws.send(WsMessage::Ws(Box::new(client)))?;
info!("Reconnected to: {}", &self.info.endpoint);
Ok(())
}
}
impl Drop for Connection {
fn drop(&mut self) {
info!("Disconnected");
}
}
fn generate_url(endpoint: &str) -> Result<Url> {
Url::parse(&format!("wss://{endpoint}/?v={VOICE_GATEWAY_VERSION}")).or(Err(Error::EndpointUrl))
}
#[inline]
async fn init_cipher(
client: &mut WsStream,
info: &ConnectionInfo,
mode: CryptoMode,
tx: &Sender<WsMessage>,
) -> Result<(Cipher, Option<davey::DaveSession>, AtomicU16)> {
loop {
let Some(value) = client.recv_event().await? else {
continue;
};
match value {
GatewayEvent::SessionDescription(desc) => {
if desc.mode != mode.to_request_str() {
return Err(Error::CryptoModeInvalid);
}
let dave_session =
if let Some(version) = NonZeroU16::new(desc.dave_protocol_version) {
let mut session = davey::DaveSession::new(
version,
info.user_id.0.into(),
info.channel_id.0.into(),
None,
)
.map_err(Error::DaveInitializationError)?;
client
.send_binary(&GatewayEvent::DaveMlsKeyPackage(DaveMlsKeyPackage {
key_package: session
.create_key_package()
.map_err(Error::DaveCreateKeyPackageError)?,
}))
.await?;
Some(session)
} else {
None
};
return Ok((
mode.cipher_from_key(&desc.secret_key)
.map_err(|_| Error::CryptoInvalidLength)?,
dave_session,
AtomicU16::new(desc.dave_protocol_version),
));
},
other => {
// Discord can and will send user-specific payload packets during this time
// which are needed to map SSRCs to `UserId`s.
tx.send(WsMessage::Deliver(other))?;
},
}
}
}

578
app/vendor/songbird/src/driver/crypto.rs vendored Normal file
View File

@@ -0,0 +1,578 @@
//! Encryption schemes supported by Discord's secure RTP negotiation.
#[cfg(any(feature = "receive", test))]
use super::tasks::error::Error as InternalError;
use aead::AeadCore;
use aes_gcm::{AeadInPlace, Aes256Gcm, Error as CryptoError};
use byteorder::{NetworkEndian, WriteBytesExt};
use chacha20poly1305::XChaCha20Poly1305;
use crypto_common::{InvalidLength, KeyInit};
#[cfg(feature = "receive")]
use discortp::rtcp::MutableRtcpPacket;
use discortp::MutablePacket;
#[cfg(any(feature = "receive", test))]
use discortp::{
rtp::{MutableRtpPacket, RtpExtensionPacket},
Packet,
};
use std::{num::Wrapping, str::FromStr};
use typenum::Unsigned;
use crate::error::ConnectionError;
/// Encryption schemes supportd by Discord.
#[derive(Clone, Copy, Debug, Eq, PartialEq, Default, Hash)]
#[non_exhaustive]
pub enum CryptoMode {
#[default]
/// Discord's currently preferred non-E2EE encryption scheme.
///
/// Packets are encrypted and decrypted using the `AES256GCM` encryption scheme.
/// An additional random 4B suffix is used as the source of nonce bytes for the packet.
/// This nonce value increments by `1` with each packet.
///
/// Encrypted content begins *after* the RTP header and extensions, following the SRTP
/// specification.
///
/// Nonce width of 4B (32b), at an extra 4B per packet (~0.2 kB/s).
Aes256Gcm,
/// A fallback non-E2EE encryption scheme.
///
/// Packets are encrypted and decrypted using the `XChaCha20Poly1305` encryption scheme.
/// An additional random 4B suffix is used as the source of nonce bytes for the packet.
/// This nonce value increments by `1` with each packet.
///
/// Encrypted content begins *after* the RTP header and extensions, following the SRTP
/// specification.
///
/// Nonce width of 4B (32b), at an extra 4B per packet (~0.2 kB/s).
XChaCha20Poly1305,
}
impl From<CryptoState> for CryptoMode {
fn from(val: CryptoState) -> Self {
match val {
CryptoState::Aes256Gcm(_) => Self::Aes256Gcm,
CryptoState::XChaCha20Poly1305(_) => Self::XChaCha20Poly1305,
}
}
}
/// The input string could not be parsed as an encryption scheme supported by songbird.
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
pub struct UnrecognisedCryptoMode;
impl FromStr for CryptoMode {
type Err = UnrecognisedCryptoMode;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"aead_aes256_gcm_rtpsize" => Ok(Self::Aes256Gcm),
"aead_xchacha20_poly1305_rtpsize" => Ok(Self::XChaCha20Poly1305),
_ => Err(UnrecognisedCryptoMode),
}
}
}
impl CryptoMode {
/// Returns the underlying crypto algorithm used by a given [`CryptoMode`].
#[must_use]
pub(crate) const fn algorithm(self) -> EncryptionAlgorithm {
match self {
CryptoMode::Aes256Gcm => EncryptionAlgorithm::Aes256Gcm,
CryptoMode::XChaCha20Poly1305 => EncryptionAlgorithm::XChaCha20Poly1305,
}
}
/// Returns an encryption cipher based on the supplied key.
///
/// Creation fails if the key is the incorrect length for the target cipher.
pub(crate) fn cipher_from_key(self, key: &[u8]) -> Result<Cipher, InvalidLength> {
match self.algorithm() {
EncryptionAlgorithm::Aes256Gcm => Aes256Gcm::new_from_slice(key)
.map(Box::new)
.map(Cipher::Aes256Gcm),
EncryptionAlgorithm::XChaCha20Poly1305 =>
XChaCha20Poly1305::new_from_slice(key).map(Cipher::XChaCha20Poly1305),
}
}
/// Returns a local priority score for a given [`CryptoMode`].
///
/// Higher values are preferred.
#[must_use]
pub(crate) fn priority(self) -> u64 {
match self {
CryptoMode::Aes256Gcm => 1,
CryptoMode::XChaCha20Poly1305 => 0,
}
}
/// Returns the best available crypto mode, given the `modes` offered by the Discord voice server.
///
/// If `preferred` is set and the mode exists in the server's supported algorithms, then that
/// mode will be chosen. Otherwise we select the highest-scoring option which is mutually understood.
pub(crate) fn negotiate<It, T>(
modes: It,
preferred: Option<Self>,
) -> Result<Self, ConnectionError>
where
T: AsRef<str>,
It: IntoIterator<Item = T>,
{
let mut best = None;
for el in modes {
let Ok(el) = CryptoMode::from_str(el.as_ref()) else {
// Unsupported mode. Ignore.
continue;
};
let mut el_priority = el.priority();
if let Some(preferred) = preferred {
if el == preferred {
el_priority = u64::MAX;
}
}
let accept = match best {
None => true,
Some((_, score)) if el_priority > score => true,
_ => false,
};
if accept {
best = Some((el, el_priority));
}
}
best.map(|(v, _)| v)
.ok_or(ConnectionError::CryptoModeUnavailable)
}
/// Returns the name of a mode as it will appear during negotiation.
#[must_use]
pub const fn to_request_str(self) -> &'static str {
match self {
Self::Aes256Gcm => "aead_aes256_gcm_rtpsize",
Self::XChaCha20Poly1305 => "aead_xchacha20_poly1305_rtpsize",
}
}
/// Returns the nonce length in bytes required by algorithm.
#[must_use]
pub const fn algorithm_nonce_size(self) -> usize {
use typenum::Unsigned as _;
match self {
Self::XChaCha20Poly1305 => <XChaCha20Poly1305 as AeadCore>::NonceSize::USIZE, // => 24
Self::Aes256Gcm => <Aes256Gcm as AeadCore>::NonceSize::USIZE, // => 12
}
}
/// Returns the number of bytes each nonce is stored as within
/// a packet.
#[must_use]
pub const fn nonce_size(self) -> usize {
match self {
Self::Aes256Gcm | Self::XChaCha20Poly1305 => 4,
}
}
/// Returns the number of bytes occupied by the encryption scheme
/// which fall before the payload.
///
/// Method name duplicated until v0.5, to prevent breaking change.
#[must_use]
pub(crate) const fn payload_prefix_len(self) -> usize {
match self {
CryptoMode::Aes256Gcm | CryptoMode::XChaCha20Poly1305 => 0,
}
}
/// Returns the tag length in bytes.
#[must_use]
pub(crate) const fn encryption_tag_len(self) -> usize {
self.algorithm().encryption_tag_len()
}
/// Returns the number of bytes occupied by the encryption scheme
/// which fall after the payload.
#[must_use]
pub const fn payload_suffix_len(self) -> usize {
self.nonce_size() + self.encryption_tag_len()
}
/// Returns the number of bytes occupied by an encryption scheme's tag which
/// fall *after* the payload.
#[must_use]
pub const fn tag_suffix_len(self) -> usize {
self.encryption_tag_len()
}
/// Calculates the number of additional bytes required compared
/// to an unencrypted payload.
#[must_use]
pub const fn payload_overhead(self) -> usize {
self.payload_prefix_len() + self.payload_suffix_len()
}
/// Extracts the byte slice in a packet used as the nonce, and the remaining mutable
/// portion of the packet.
fn nonce_slice<'a>(
self,
_header: &'a [u8],
body: &'a mut [u8],
) -> Result<(&'a [u8], &'a mut [u8]), CryptoError> {
match self {
Self::Aes256Gcm | Self::XChaCha20Poly1305 => {
let len = body.len();
if len < self.payload_suffix_len() {
Err(CryptoError)
} else {
let (body_left, nonce_loc) = body.split_at_mut(len - self.nonce_size());
Ok((nonce_loc, body_left))
}
},
}
}
}
/// State used in nonce generation for the encryption variants in [`CryptoMode`].
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum CryptoState {
/// An additional random 4B suffix is used as the source of nonce bytes for the packet.
/// This nonce value increments by `1` with each packet.
///
/// The last used nonce is stored.
Aes256Gcm(Wrapping<u32>),
/// An additional random 4B suffix is used as the source of nonce bytes for the packet.
/// This nonce value increments by `1` with each packet.
///
/// The last used nonce is stored.
XChaCha20Poly1305(Wrapping<u32>),
}
impl From<CryptoMode> for CryptoState {
fn from(val: CryptoMode) -> Self {
match val {
CryptoMode::Aes256Gcm => CryptoState::Aes256Gcm(Wrapping(rand::random::<u32>())),
CryptoMode::XChaCha20Poly1305 =>
CryptoState::XChaCha20Poly1305(Wrapping(rand::random::<u32>())),
}
}
}
impl CryptoState {
/// Writes packet nonce into the body, if required, returning the new length.
pub fn write_packet_nonce(
&mut self,
packet: &mut impl MutablePacket,
payload_end: usize,
) -> usize {
let mode = self.kind();
let endpoint = payload_end + mode.payload_suffix_len();
let startpoint = endpoint - mode.nonce_size();
match self {
Self::Aes256Gcm(ref mut i) | Self::XChaCha20Poly1305(ref mut i) => {
(&mut packet.payload_mut()[startpoint..endpoint])
.write_u32::<NetworkEndian>(i.0)
.expect(
"Nonce size is guaranteed to be sufficient to write u32 for lite tagging.",
);
*i += Wrapping(1);
},
}
endpoint
}
/// Returns the underlying (stateless) type of the active crypto mode.
#[must_use]
pub fn kind(self) -> CryptoMode {
CryptoMode::from(self)
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
pub(crate) enum EncryptionAlgorithm {
Aes256Gcm,
XChaCha20Poly1305,
}
impl EncryptionAlgorithm {
#[must_use]
pub(crate) const fn encryption_tag_len(self) -> usize {
match self {
Self::Aes256Gcm => <Aes256Gcm as AeadCore>::TagSize::USIZE, // 16
Self::XChaCha20Poly1305 => <XChaCha20Poly1305 as AeadCore>::TagSize::USIZE, // 16
}
}
}
impl From<&Cipher> for EncryptionAlgorithm {
fn from(value: &Cipher) -> Self {
match value {
Cipher::XChaCha20Poly1305(_) => EncryptionAlgorithm::XChaCha20Poly1305,
Cipher::Aes256Gcm(_) => EncryptionAlgorithm::Aes256Gcm,
}
}
}
#[derive(Clone)]
pub enum Cipher {
XChaCha20Poly1305(XChaCha20Poly1305),
Aes256Gcm(Box<Aes256Gcm>),
}
impl Cipher {
#[must_use]
pub(crate) fn mode(&self) -> CryptoMode {
match self {
Cipher::XChaCha20Poly1305(_) => CryptoMode::XChaCha20Poly1305,
Cipher::Aes256Gcm(_) => CryptoMode::Aes256Gcm,
}
}
#[must_use]
pub(crate) fn encryption_tag_len(&self) -> usize {
EncryptionAlgorithm::from(self).encryption_tag_len()
}
/// Encrypts a Discord RT(C)P packet using the given key.
///
/// Use of this requires that the input packet has had a nonce generated in the correct location,
/// and `payload_len` specifies the number of bytes after the header including this nonce.
#[inline]
pub fn encrypt_pkt_in_place(
&self,
packet: &mut impl MutablePacket,
payload_len: usize,
) -> Result<(), CryptoError> {
let mode = self.mode();
let header_len = packet.packet().len() - packet.payload().len();
let (header, body) = packet.packet_mut().split_at_mut(header_len);
let (slice_to_use, body_remaining) = mode.nonce_slice(header, &mut body[..payload_len])?;
let tag_size = self.encryption_tag_len();
// body_remaining is now correctly truncated to exclude the nonce by this point.
// the true_payload to encrypt is within the buf[prefix:-suffix].
let (_, body_remaining) = body_remaining.split_at_mut(mode.payload_prefix_len());
let (body, post_payload) =
body_remaining.split_at_mut(body_remaining.len() - mode.tag_suffix_len());
// All these Nonce types are distinct at the type level
// (96b for AES, 192b for XChaCha).
match self {
// The below variants follow part of the SRTP spec (RFC3711, sec 3.1)
// by requiring that we include the cleartext header portion as
// authenticated data.
Self::Aes256Gcm(aes_gcm) => {
let mut nonce = aes_gcm::Nonce::default();
nonce[..mode.nonce_size()].copy_from_slice(slice_to_use);
let tag = aes_gcm.encrypt_in_place_detached(&nonce, header, body)?;
post_payload[..tag_size].copy_from_slice(&tag[..]);
},
Self::XChaCha20Poly1305(cha_cha_poly1305) => {
let mut nonce = chacha20poly1305::XNonce::default();
nonce[..mode.nonce_size()].copy_from_slice(slice_to_use);
let tag = cha_cha_poly1305.encrypt_in_place_detached(&nonce, header, body)?;
post_payload[..tag_size].copy_from_slice(&tag[..]);
},
}
Ok(())
}
#[cfg(any(feature = "receive", test))]
pub(crate) fn decrypt_rtp_in_place(
&self,
packet: &mut MutableRtpPacket<'_>,
) -> Result<(usize, usize), InternalError> {
// An exciting difference from the SRTP spec: Discord begins encryption
// after the RTP extension *header*, encrypting the extensions themselves,
// whereas the spec leaves all extensions in the clear.
// This header is described as the 'extension preamble'.
let has_extension = packet.get_extension() != 0;
let plain_bytes = if has_extension {
// CSRCs and extension bytes will be in the plaintext segment.
// We will need these demarcated to select the right bytes to
// decrypt, and to use as auth data.
RtpExtensionPacket::minimum_packet_size()
} else {
0
};
let (_, end) = self.decrypt_pkt_in_place(packet, plain_bytes)?;
// Update the start estimate to account for bytes occupied by extension headers.
let payload_offset = if has_extension {
let payload = packet.payload();
let extension =
RtpExtensionPacket::new(payload).ok_or(InternalError::IllegalVoicePacket)?;
extension.packet().len() - extension.payload().len()
} else {
0
};
Ok((payload_offset, end))
}
#[cfg(feature = "receive")]
pub(crate) fn decrypt_rtcp_in_place(
&self,
packet: &mut MutableRtcpPacket<'_>,
) -> Result<(usize, usize), InternalError> {
// RTCP/SRTCP have identical handling -- no var-length elements
// are included as part of the plaintext.
self.decrypt_pkt_in_place(packet, 0)
}
/// Decrypts an arbitrary packet using the given key.
///
/// If successful, this returns the number of bytes to be ignored from the
/// start and end of the packet payload.
#[inline]
#[cfg(any(feature = "receive", test))]
pub(crate) fn decrypt_pkt_in_place(
&self,
packet: &mut impl MutablePacket,
n_plaintext_body_bytes: usize,
) -> Result<(usize, usize), InternalError> {
let mode = self.mode();
let header_len = packet.packet().len() - packet.payload().len();
let plaintext_end = header_len + n_plaintext_body_bytes;
let (plaintext, ciphertext) = packet
.packet_mut()
.split_at_mut_checked(plaintext_end)
.ok_or(CryptoError)?;
let (slice_to_use, body_remaining) = mode.nonce_slice(plaintext, ciphertext)?;
let (pre_payload, body_remaining) = body_remaining
.split_at_mut_checked(mode.payload_prefix_len())
.ok_or(CryptoError)?;
let suffix_split_point = body_remaining
.len()
.checked_sub(mode.tag_suffix_len())
.ok_or(CryptoError)?;
let (body, post_payload) = body_remaining
.split_at_mut_checked(suffix_split_point)
.ok_or(CryptoError)?;
let tag_size = self.encryption_tag_len();
match self {
// The below variants follow part of the SRTP spec (RFC3711, sec 3.1)
// by requiring that we include the cleartext header portion as
// authenticated data.
Self::Aes256Gcm(aes_gcm) => {
let mut nonce = aes_gcm::Nonce::default();
nonce[..mode.nonce_size()].copy_from_slice(slice_to_use);
let tag = aes_gcm::Tag::from_slice(&post_payload[..tag_size]);
aes_gcm.decrypt_in_place_detached(&nonce, plaintext, body, tag)?;
},
Self::XChaCha20Poly1305(cha_cha_poly1305) => {
let mut nonce = chacha20poly1305::XNonce::default();
nonce[..mode.nonce_size()].copy_from_slice(slice_to_use);
let tag = chacha20poly1305::Tag::from_slice(&post_payload[..tag_size]);
cha_cha_poly1305.decrypt_in_place_detached(&nonce, plaintext, body, tag)?;
},
}
Ok((
plaintext_end + pre_payload.len(),
post_payload.len() + slice_to_use.len(),
))
}
}
#[cfg(test)]
mod test {
use super::*;
use discortp::rtp::MutableRtpPacket;
#[test]
fn small_packet_decrypts_error() {
let mut buf = [0u8; MutableRtpPacket::minimum_packet_size()];
let modes = [CryptoMode::Aes256Gcm, CryptoMode::XChaCha20Poly1305];
let mut pkt = MutableRtpPacket::new(&mut buf[..]).unwrap();
for mode in modes {
// Coincidentally, these are all 32B for now.
let cipher = mode.cipher_from_key(&[1u8; 32]).unwrap();
// AIM: should error, and not panic.
assert!(cipher.decrypt_rtp_in_place(&mut pkt).is_err());
}
}
#[test]
fn symmetric_encrypt_decrypt_tag_after_data() {
const TRUE_PAYLOAD: [u8; 8] = [1, 2, 3, 4, 5, 6, 7, 8];
for mode in [CryptoMode::Aes256Gcm, CryptoMode::XChaCha20Poly1305] {
let mut buf = vec![
0u8;
MutableRtpPacket::minimum_packet_size()
+ TRUE_PAYLOAD.len()
+ mode.nonce_size()
+ mode.encryption_tag_len()
];
buf.fill(0);
let cipher = mode.cipher_from_key(&[7u8; 32]).unwrap();
let mut pkt = MutableRtpPacket::new(&mut buf[..]).unwrap();
let mut crypto_state = CryptoState::from(mode);
let payload = pkt.payload_mut();
payload[mode.payload_prefix_len()..TRUE_PAYLOAD.len()].copy_from_slice(&TRUE_PAYLOAD);
let final_payload_size = crypto_state.write_packet_nonce(&mut pkt, TRUE_PAYLOAD.len());
let enc_succ = cipher.encrypt_pkt_in_place(&mut pkt, final_payload_size);
assert!(enc_succ.is_ok());
let final_pkt_len = MutableRtpPacket::minimum_packet_size() + final_payload_size;
let mut pkt = MutableRtpPacket::new(&mut buf[..final_pkt_len]).unwrap();
assert!(cipher.decrypt_rtp_in_place(&mut pkt).is_ok());
}
}
#[test]
fn negotiate_cryptomode() {
// If we have no preference (or our preference is missing), choose the highest available in the set.
let test_set =
[CryptoMode::XChaCha20Poly1305, CryptoMode::Aes256Gcm].map(CryptoMode::to_request_str);
assert_eq!(
CryptoMode::negotiate(test_set, None).unwrap(),
CryptoMode::Aes256Gcm
);
let test_set_missing = [CryptoMode::XChaCha20Poly1305].map(CryptoMode::to_request_str);
assert_eq!(
CryptoMode::negotiate(test_set_missing, Some(CryptoMode::Aes256Gcm)).unwrap(),
CryptoMode::XChaCha20Poly1305
);
// Preference wins in spite of the defined `priority` value.
assert_eq!(
CryptoMode::negotiate(test_set, Some(CryptoMode::XChaCha20Poly1305)).unwrap(),
CryptoMode::XChaCha20Poly1305
);
// If there is no mutual intelligibility, return an error.
let bad_modes = ["not_real", "des", "rc5"];
assert!(CryptoMode::negotiate(bad_modes, None).is_err());
assert!(CryptoMode::negotiate(bad_modes, Some(CryptoMode::Aes256Gcm)).is_err());
}
}

View File

@@ -0,0 +1,125 @@
use opus2::Channels as OpusChannels;
/// Decode behaviour for received RTP packets within the driver.
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
#[non_exhaustive]
pub enum DecodeMode {
/// Packets received from Discord are handed over to events without any
/// changes applied.
///
/// No CPU work involved.
Pass,
/// Decrypts the body of each received packet.
///
/// Small per-packet CPU use.
Decrypt,
/// Decrypts and decodes each received packet, correctly accounting for losses.
///
/// Larger per-packet CPU use.
Decode(DecodeConfig),
}
impl DecodeMode {
/// Returns whether this mode will decrypt and decode received packets.
#[must_use]
pub fn should_decode(self) -> bool {
matches!(self, DecodeMode::Decode(..))
}
/// Returns whether this mode will decrypt received packets.
#[must_use]
pub fn should_decrypt(self) -> bool {
self != DecodeMode::Pass
}
}
/// Configuration for [`DecodeMode::Decode`]
#[non_exhaustive]
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash)]
pub struct DecodeConfig {
/// Configures the channel layout for output audio.
///
/// Defaults to [`Channels::Stereo`].
pub channels: Channels,
/// Configures the sample rate for output audio.
///
/// Defaults to [`SampleRate::Hz48000`].
pub sample_rate: SampleRate,
}
impl DecodeConfig {
/// Creates a new [`DecodeConfig`] with the specified channels and sample rate.
#[must_use]
pub fn new(channels: Channels, sample_rate: SampleRate) -> Self {
Self {
channels,
sample_rate,
}
}
}
/// The channel layout of output audio when using [`DecodeMode::Decode`].
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash)]
#[non_exhaustive]
pub enum Channels {
/// Decode received audio packets into a single channel.
Mono,
/// Decode received audio packets into two interleaved channels.
///
/// Received mono packets' samples will automatically be duplicated across
/// both channels.
///
/// The default choice.
#[default]
Stereo,
}
impl Channels {
pub(crate) fn channels(self) -> usize {
match self {
Channels::Mono => 1,
Channels::Stereo => 2,
}
}
}
impl From<Channels> for OpusChannels {
fn from(value: Channels) -> Self {
match value {
Channels::Mono => OpusChannels::Mono,
Channels::Stereo => OpusChannels::Stereo,
}
}
}
/// The sample rate of output audio when using [`DecodeMode::Decode`].
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash)]
#[non_exhaustive]
pub enum SampleRate {
/// Decode to a sample rate of 8kHz.
Hz8000,
/// Decode to a sample rate of 12kHz.
Hz12000,
/// Decode to a sample rate of 16kHz.
Hz16000,
/// Decode to a sample rate of 24kHz.
Hz24000,
/// Decode to a sample rate of 48kHz.
///
/// The preferred option for encoding/decoding at or above CD quality.
#[default]
Hz48000,
}
impl From<SampleRate> for u32 {
fn from(value: SampleRate) -> Self {
match value {
SampleRate::Hz8000 => 8000,
SampleRate::Hz12000 => 12000,
SampleRate::Hz16000 => 16000,
SampleRate::Hz24000 => 24000,
SampleRate::Hz48000 => 48000,
}
}
}

View File

@@ -0,0 +1,59 @@
use opus2::Channels;
use symphonia_core::audio::Layout;
use crate::constants::{MONO_FRAME_SIZE, STEREO_FRAME_SIZE};
/// Mixing behaviour for sent audio sources processed within the driver.
///
/// This has no impact on Opus packet passthrough, which will pass packets
/// irrespective of their channel count.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum MixMode {
/// Audio sources will be downmixed into a mono buffer.
Mono,
/// Audio sources will be mixed into into a stereo buffer, where mono sources
/// will be duplicated into both channels.
Stereo,
}
impl MixMode {
pub(crate) const fn to_opus(self) -> Channels {
match self {
Self::Mono => Channels::Mono,
Self::Stereo => Channels::Stereo,
}
}
pub(crate) const fn sample_count_in_frame(self) -> usize {
match self {
Self::Mono => MONO_FRAME_SIZE,
Self::Stereo => STEREO_FRAME_SIZE,
}
}
pub(crate) const fn channels(self) -> usize {
match self {
Self::Mono => 1,
Self::Stereo => 2,
}
}
pub(crate) const fn symph_layout(self) -> Layout {
match self {
Self::Mono => Layout::Mono,
Self::Stereo => Layout::Stereo,
}
}
}
impl From<MixMode> for Layout {
fn from(val: MixMode) -> Self {
val.symph_layout()
}
}
impl From<MixMode> for Channels {
fn from(val: MixMode) -> Self {
val.to_opus()
}
}

357
app/vendor/songbird/src/driver/mod.rs vendored Normal file
View File

@@ -0,0 +1,357 @@
//! Runner for a voice connection.
//!
//! Songbird's driver is a mixed-sync system, using:
//! * Asynchronous connection management, event-handling, and gateway integration.
//! * Synchronous audio mixing, packet generation, and encoding.
//!
//! This splits up work according to its IO/compute bound nature, preventing packet
//! generation from being slowed down past its deadline, or from affecting other
//! asynchronous tasks your bot must handle.
#[cfg(feature = "internals")]
pub mod bench_internals;
pub(crate) mod connection;
mod crypto;
#[cfg(feature = "receive")]
mod decode_mode;
mod mix_mode;
pub mod retry;
mod scheduler;
pub(crate) mod tasks;
#[cfg(test)]
pub(crate) mod test_config;
#[cfg(any(test, feature = "internals"))]
mod test_impls;
use connection::error::{Error, Result};
pub use crypto::CryptoMode;
pub(crate) use crypto::CryptoState;
#[cfg(feature = "receive")]
pub use decode_mode::*;
pub use mix_mode::MixMode;
pub use scheduler::{
get_default_scheduler,
Config as SchedulerConfig,
Error as SchedulerError,
LiveStatBlock,
Mode as SchedulerMode,
Scheduler,
};
#[cfg(test)]
pub use test_config::*;
#[cfg(any(test, feature = "internals"))]
pub use test_impls::*;
#[cfg(feature = "builtin-queue")]
use crate::tracks::TrackQueue;
use crate::{
events::EventData,
input::Input,
tracks::{Track, TrackHandle},
Config,
ConnectionInfo,
Event,
EventHandler,
};
use core::{
future::Future,
pin::Pin,
task::{Context, Poll},
};
use flume::{r#async::RecvFut, SendError, Sender};
/// Opus encoder bitrate settings.
pub use opus2::{self as opus, Bitrate};
#[cfg(feature = "builtin-queue")]
use std::time::Duration;
#[allow(unused_imports)]
pub use tasks::disposal::DisposalThread;
use tasks::message::CoreMessage;
use tracing::instrument;
/// The control object for a Discord voice connection, handling connection,
/// mixing, encoding, en/decryption, and event generation.
///
/// When compiled with the `"builtin-queue"` feature, each driver includes a track queue
/// as a convenience to prevent the additional overhead of per-guild state management.
#[derive(Clone, Debug)]
pub struct Driver {
config: Config,
self_mute: bool,
sender: Sender<CoreMessage>,
// Making this an Option is an abhorrent hack to coerce the borrow checker
// into letting us have an &TrackQueue at the same time as an &mut Driver.
// This is probably preferable to cloning the driver: Arc<...> should be nonzero
// and if the compiler's smart we'll just codegen a pointer swap. It definitely makes
// use of NonZero.
#[cfg(feature = "builtin-queue")]
queue: Option<TrackQueue>,
}
impl Driver {
/// Creates a new voice driver.
///
/// This will create the core voice tasks in the background.
#[inline]
#[must_use]
pub fn new(config: Config) -> Self {
let sender = Self::start_inner(config.clone());
Driver {
config,
self_mute: false,
sender,
#[cfg(feature = "builtin-queue")]
queue: Some(TrackQueue::default()),
}
}
fn start_inner(config: Config) -> Sender<CoreMessage> {
let (tx, rx) = flume::unbounded();
tasks::start(config, rx, tx.clone());
tx
}
fn restart_inner(&mut self) {
self.sender = Self::start_inner(self.config.clone());
self.mute(self.self_mute);
}
/// Connects to a voice channel using the specified server.
///
/// This method instantly contacts the driver tasks, and its
/// does not need to be `await`ed to start the actual connection.
#[instrument(skip(self))]
pub fn connect(&mut self, info: ConnectionInfo) -> Connect {
let (tx, rx) = flume::bounded(1);
self.raw_connect(info, tx);
Connect {
inner: rx.into_recv_async(),
}
}
/// Connects to a voice channel using the specified server.
#[instrument(skip(self))]
pub(crate) fn raw_connect(&mut self, info: ConnectionInfo, tx: Sender<Result<()>>) {
self.send(CoreMessage::ConnectWithResult(info, tx));
}
/// Leaves the current voice channel, disconnecting from it.
///
/// This does *not* forget settings, like whether to be self-deafened or
/// self-muted.
#[instrument(skip(self))]
pub fn leave(&mut self) {
self.send(CoreMessage::Disconnect);
}
/// Sets whether the current connection is to be muted.
///
/// If there is no live voice connection, then this only acts as a settings
/// update for future connections.
#[instrument(skip(self))]
pub fn mute(&mut self, mute: bool) {
self.self_mute = mute;
self.send(CoreMessage::Mute(mute));
}
/// Returns whether the driver is muted (i.e., processes audio internally
/// but submits none).
#[instrument(skip(self))]
pub fn is_mute(&self) -> bool {
self.self_mute
}
/// Plays audio from an input, returning a handle for further control.
#[instrument(skip(self, input))]
pub fn play_input(&mut self, input: Input) -> TrackHandle {
self.play(input.into())
}
/// Plays audio from an input, returning a handle for further control.
///
/// Unlike [`Self::play_input`], this stops all other inputs attached
/// to the channel.
#[instrument(skip(self, input))]
pub fn play_only_input(&mut self, input: Input) -> TrackHandle {
self.play_only(input.into())
}
/// Plays audio from a [`Track`] object.
///
/// The main difference between this function and [`Self::play_input`] is
/// that this allows for direct manipulation of the [`Track`] object
/// before it is passed over to the voice and mixing contexts.
#[instrument(skip(self, track))]
pub fn play(&mut self, track: Track) -> TrackHandle {
let (handle, ctx) = track.into_context();
self.send(CoreMessage::AddTrack(Box::new(ctx)));
handle
}
/// Exclusively plays audio from a [`Track`] object.
///
/// As in [`Self::play_only_input`], this stops all other sources attached to the
/// channel. Like [`Self::play`], however, this allows for direct manipulation of the
/// [`Track`] object before it is passed over to the voice and mixing contexts.
#[instrument(skip(self, track))]
pub fn play_only(&mut self, track: Track) -> TrackHandle {
let (handle, ctx) = track.into_context();
self.send(CoreMessage::SetTrack(Some(Box::new(ctx))));
handle
}
/// Sets the bitrate for encoding Opus packets sent along
/// the channel being managed.
///
/// The default rate is 128 kbps.
/// Sensible values range between `Bits(512)` and `Bits(512_000)`
/// bits per second.
/// Alternatively, `Auto` and `Max` remain available.
#[instrument(skip(self))]
pub fn set_bitrate(&mut self, bitrate: Bitrate) {
self.send(CoreMessage::SetBitrate(bitrate));
}
/// Stops playing audio from all sources, if any are set.
#[instrument(skip(self))]
pub fn stop(&mut self) {
self.send(CoreMessage::SetTrack(None));
}
/// Sets the configuration for this driver (and parent `Call`, if applicable).
#[instrument(skip(self))]
pub fn set_config(&mut self, config: Config) {
self.config = config.clone();
self.send(CoreMessage::SetConfig(config));
}
/// Returns a view of this driver's configuration.
#[instrument(skip(self))]
pub fn config(&self) -> &Config {
&self.config
}
/// Attach a global event handler to an audio context. Global events may receive
/// any [`EventContext`].
///
/// Global timing events will tick regardless of whether audio is playing,
/// so long as the bot is connected to a voice channel, and have no tracks.
/// [`TrackEvent`]s will respond to all relevant tracks, giving some audio elements.
///
/// Users **must** ensure that no costly work or blocking occurs
/// within the supplied function or closure. *Taking excess time could prevent
/// timely sending of packets, causing audio glitches and delays*.
///
/// [`TrackEvent`]: crate::events::TrackEvent
/// [`EventContext`]: crate::events::EventContext
#[instrument(skip(self, action))]
pub fn add_global_event<F: EventHandler + 'static>(&mut self, event: Event, action: F) {
self.send(CoreMessage::AddEvent(EventData::new(event, action)));
}
/// Removes all global event handlers from an audio context.
#[instrument(skip(self))]
pub fn remove_all_global_events(&mut self) {
self.send(CoreMessage::RemoveGlobalEvents);
}
/// Sends a message to the inner tasks, restarting it if necessary.
fn send(&mut self, status: CoreMessage) {
// Restart thread if it errored.
if let Err(SendError(status)) = self.sender.send(status) {
self.restart_inner();
self.sender.send(status).unwrap();
}
}
}
#[cfg(feature = "builtin-queue")]
impl Driver {
/// Returns a reference to this driver's built-in queue.
///
/// Requires the `"builtin-queue"` feature.
/// Queue additions should be made via [`Driver::enqueue`] and
/// [`Driver::enqueue_input`].
#[must_use]
pub fn queue(&self) -> &TrackQueue {
self.queue
.as_ref()
.expect("Queue: The only case this can fail is if a previous queue operation panicked.")
}
/// Adds an audio [`Input`] to this driver's built-in queue.
///
/// Requires the `"builtin-queue"` feature.
pub async fn enqueue_input(&mut self, input: Input) -> TrackHandle {
self.enqueue(input.into()).await
}
/// Adds an existing [`Track`] to this driver's built-in queue.
///
/// Requires the `"builtin-queue"` feature.
pub async fn enqueue(&mut self, mut track: Track) -> TrackHandle {
let preload_time = TrackQueue::get_preload_time(&mut track).await;
self.enqueue_with_preload(track, preload_time)
}
/// Add an existing [`Track`] to the queue, using a known time to preload the next track.
///
/// See [`TrackQueue::add_with_preload`] for how `preload_time` is used.
///
/// Requires the `"builtin-queue"` feature.
pub fn enqueue_with_preload(
&mut self,
track: Track,
preload_time: Option<Duration>,
) -> TrackHandle {
let queue = self.queue.take().expect(
"Enqueue: The only case this can fail is if a previous queue operation panicked.",
);
let handle = queue.add_with_preload(track, self, preload_time);
self.queue = Some(queue);
handle
}
}
impl Default for Driver {
fn default() -> Self {
Self::new(Config::default())
}
}
impl Drop for Driver {
/// Leaves the current connected voice channel, if connected to one, and
/// forgets all configurations relevant to this Handler.
fn drop(&mut self) {
drop(self.sender.send(CoreMessage::Poison));
}
}
/// Future for a call to [`Driver::connect`].
///
/// This future awaits the *result* of a connection; the driver
/// is messaged at the time of the call.
pub struct Connect {
inner: RecvFut<'static, Result<()>>,
}
impl Future for Connect {
type Output = Result<()>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
match Pin::new(&mut self.inner).poll(cx) {
Poll::Ready(r) => Poll::Ready(r.map_err(|_| Error::AttemptDiscarded).and_then(|x| x)),
Poll::Pending => Poll::Pending,
}
}
}

View File

@@ -0,0 +1,51 @@
//! Configuration for connection retries.
mod strategy;
pub use self::strategy::*;
use std::num::NonZeroU8;
use crate::FloatDuration;
/// Configuration to be used for retrying driver connection attempts.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Retry {
/// Strategy used to determine how long to wait between retry attempts.
///
/// *Defaults to an [`ExponentialBackoff`] from 0.25s
/// to 10s, with a jitter of `0.1`.*
///
/// [`ExponentialBackoff`]: Strategy::Backoff
pub strategy: Strategy,
/// The maximum number of retries to attempt.
///
/// `None` will attempt an infinite number of retries,
/// while `Some(0)` will attempt to connect *once* (no retries).
///
/// *Defaults to `Some(5)`.*
pub retry_limit: Option<NonZeroU8>,
}
impl Default for Retry {
fn default() -> Self {
Self {
strategy: Strategy::Backoff(ExponentialBackoff::default()),
retry_limit: Some(const { NonZeroU8::new(5).unwrap() }),
}
}
}
impl Retry {
pub(crate) fn retry_in(
&self,
last_wait: Option<FloatDuration>,
attempts: u8,
) -> Option<FloatDuration> {
if self.retry_limit.is_none_or(|a| attempts < a.get()) {
Some(self.strategy.retry_in(last_wait))
} else {
None
}
}
}

View File

@@ -0,0 +1,77 @@
use std::time::Duration;
use rand::random;
use crate::FloatDuration;
/// Logic used to determine how long to wait between retry attempts.
#[derive(Clone, Copy, Debug, PartialEq)]
#[non_exhaustive]
pub enum Strategy {
/// The driver will wait for the same amount of time between each retry.
Every(FloatDuration),
/// Exponential backoff waiting strategy, where the duration between
/// attempts (approximately) doubles each time.
Backoff(ExponentialBackoff),
}
impl Strategy {
pub(crate) fn retry_in(&self, last_wait: Option<FloatDuration>) -> FloatDuration {
match self {
Self::Every(t) => *t,
Self::Backoff(exp) => exp.retry_in(last_wait),
}
}
}
/// Exponential backoff waiting strategy.
///
/// Each attempt waits for twice the last delay plus/minus a
/// random jitter, clamped to a min and max value.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct ExponentialBackoff {
/// Minimum amount of time to wait between retries.
///
/// *Defaults to 0.25s.*
pub min: FloatDuration,
/// Maximum amount of time to wait between retries.
///
/// This will be clamped to `>=` min.
///
/// *Defaults to 10s.*
pub max: FloatDuration,
/// Amount of uniform random jitter to apply to generated wait times.
/// I.e., 0.1 will add +/-10% to generated intervals.
///
/// This is restricted to within +/-100%.
///
/// *Defaults to `0.1`.*
pub jitter: f32,
}
impl Default for ExponentialBackoff {
fn default() -> Self {
Self {
min: Duration::from_millis(250).into(),
max: Duration::from_secs(10).into(),
jitter: 0.1,
}
}
}
impl ExponentialBackoff {
pub(crate) fn retry_in(&self, last_wait: Option<FloatDuration>) -> FloatDuration {
let min = self.min.as_secs_f32();
let max = self.max.as_secs_f32();
let attempt = last_wait.map_or(min, |t| 2.0 * t.as_secs_f32());
let perturb = (1.0 - (self.jitter * 2.0 * (random::<f32>() - 1.0))).clamp(0.0, 2.0);
let mut target_time = attempt * perturb;
// Now clamp target time into given range.
let safe_max = if max < min { min } else { max };
target_time = target_time.clamp(min, safe_max);
FloatDuration::from_secs_f32(target_time)
}
}

View File

@@ -0,0 +1,64 @@
use super::*;
/// Configuration for how a [`Scheduler`] handles tasks.
///
/// [`Scheduler`]: super::Scheduler
#[derive(Clone, Debug)]
#[non_exhaustive]
pub struct Config {
/// How Live mixer tasks will be mapped to individual threads.
///
/// Defaults to `Mode::MaxPerThread(16)`.
pub strategy: Mode,
/// Move costly mixers to another thread if their parent worker is at
/// risk of missing its deadlines.
///
/// Defaults to `true`.
pub move_expensive_tasks: bool,
}
impl Default for Config {
fn default() -> Self {
Self {
strategy: Mode::default(),
move_expensive_tasks: true,
}
}
}
/// Strategies for mapping live mixer tasks to individual threads.
///
/// Defaults to `MaxPerThread(16)`.
#[derive(Clone, Debug)]
#[non_exhaustive]
pub enum Mode {
/// Allows at most `n` tasks to run per thread.
MaxPerThread(NonZeroUsize),
}
impl Mode {
/// Returns the number of `Mixer`s that a scheduler should preallocate
/// resources for.
pub(crate) fn prealloc_size(&self) -> usize {
match self {
Self::MaxPerThread(n) => n.get(),
}
}
/// Returns the maximum number of concurrent mixers that a scheduler is
/// allowed to place on a single thread.
///
/// Future scheduling modes may choose to limit *only* on execution cost.
#[allow(clippy::unnecessary_wraps)]
pub(crate) fn task_limit(&self) -> Option<usize> {
match self {
Self::MaxPerThread(n) => Some(n.get()),
}
}
}
impl Default for Mode {
fn default() -> Self {
Self::MaxPerThread(DEFAULT_MIXERS_PER_THREAD)
}
}

View File

@@ -0,0 +1,335 @@
use std::{collections::HashMap, time::Duration};
use nohash_hasher::{BuildNoHashHasher, IntMap};
use tokio::time::{Instant as TokInstant, Interval};
use tracing::info;
use crate::constants::*;
use super::*;
const THREAD_CULL_TIMER: Duration = Duration::from_secs(60);
/// An async task responsible for maintaining UDP keepalives and event state for inactive
/// `Mixer` tasks.
pub(crate) struct Idle {
config: Config,
cull_timer: Duration,
tasks: IntMap<TaskId, ParkedMixer>,
// track taskids which are live to prevent their realloc? unlikely w u64 but still
pub(crate) stats: Arc<StatBlock>,
rx: Receiver<SchedulerMessage>,
tx: Sender<SchedulerMessage>,
next_id: TaskId,
next_worker_id: WorkerId,
workers: Vec<Worker>,
to_cull: Vec<TaskId>,
}
impl Idle {
pub fn new(config: Config) -> (Self, Sender<SchedulerMessage>) {
let (tx, rx) = flume::unbounded();
let stats = Arc::default();
let tasks = HashMap::with_capacity_and_hasher(128, BuildNoHashHasher::default());
// TODO: include heap of keepalive sending times?
let out = Self {
config,
cull_timer: THREAD_CULL_TIMER,
tasks,
stats,
rx,
tx: tx.clone(),
next_id: TaskId::new(),
next_worker_id: WorkerId::new(),
workers: Vec::with_capacity(16),
to_cull: vec![],
};
(out, tx)
}
/// Run the inner task until all external `Scheduler` handles are dropped.
async fn run(&mut self) {
let mut interval = tokio::time::interval(TIMESTEP_LENGTH);
while self.run_once(&mut interval).await {}
}
/// Run one 'tick' of idle thread maintenance.
///
/// This is a priority system over 2 main tasks:
/// 1) handle scheduling/upgrade/action requests for mixers
/// 2) [every 20ms]tick the main timer for each task, send keepalive if
/// needed, reclaim & cull workers.
///
/// Idle mixers spawn an async task each to forward their `MixerMessage`s
/// on to this task to be handled by 1). These tasks self-terminate if a
/// message would make a mixer `now_live`.
async fn run_once(&mut self, interval: &mut Interval) -> bool {
tokio::select! {
biased;
msg = self.rx.recv_async() => match msg {
Ok(SchedulerMessage::NewMixer(rx, ic, cfg)) => {
let mut mixer = ParkedMixer::new(rx, ic, cfg);
let id = self.next_id.incr();
mixer.spawn_forwarder(self.tx.clone(), id);
self.tasks.insert(id, mixer);
self.stats.add_idle_mixer();
},
Ok(SchedulerMessage::Demote(id, mut task)) => {
task.send_gateway_not_speaking();
task.spawn_forwarder(self.tx.clone(), id);
self.tasks.insert(id, task);
},
Ok(SchedulerMessage::Do(id, mix_msg)) => {
let maybe_live = mix_msg.is_mixer_maybe_live();
if let Some(task) = self.tasks.get_mut(&id) {
match task.handle_message(mix_msg) {
Ok(false) if maybe_live => {
if task.mixer.tracks.is_empty() {
// No tracks, likely due to SetConn.
// Recreate message forwarding task.
task.spawn_forwarder(self.tx.clone(), id);
} else {
let task = self.tasks.remove(&id).unwrap();
self.schedule_mixer(task, id, None);
}
},
Ok(false) => {},
Ok(true) | Err(()) => self.to_cull.push(id),
}
} else {
info!("Received post-cull message for {id:?}, discarding.");
}
},
Ok(SchedulerMessage::Overspill(worker_id, id, task)) => {
self.schedule_mixer(task, id, Some(worker_id));
},
Ok(SchedulerMessage::GetStats(tx)) => {
_ = tx.send(self.workers.iter().map(Worker::stats).collect());
},
Ok(SchedulerMessage::Kill) | Err(_) => {
return false;
},
},
_ = interval.tick() => {
// TODO: store keepalive sends in another data structure so
// we don't check every task every 20ms.
//
// if we can also make tick handling lazy(er), we can also optimise for that.
let now = TokInstant::now();
for (id, task) in &mut self.tasks {
// NOTE: this is a non-blocking send so safe from async context.
if task.tick_and_keepalive(now.into()).is_err() {
self.to_cull.push(*id);
}
}
let mut i = 0;
while i < self.workers.len() {
if let Some(then) = self.workers[i].try_mark_empty(now) {
if now.duration_since(then) >= self.cull_timer {
self.workers.swap_remove(i);
continue;
}
}
i += 1;
}
},
}
for id in self.to_cull.drain(..) {
if let Some(tx) = self.tasks.remove(&id).and_then(|t| t.cull_handle) {
_ = tx.send_async(()).await;
}
}
true
}
/// Promote a task to a live mixer thread.
fn schedule_mixer(&mut self, mut task: ParkedMixer, id: TaskId, avoid: Option<WorkerId>) {
if task.send_gateway_speaking().is_ok() {
// If a worker ever completely fails, then we need to remove it here
// `fetch_worker` will either find another, or generate us a new one if
// none exist.
// We need to track ownership of the task coming back via SendError using this
// Option.
let mut loop_task = Some(task);
loop {
let task = loop_task.take().unwrap();
let (worker, idx) = self.fetch_worker(&task, avoid);
match worker.schedule_mixer(id, task) {
Ok(()) => {
self.stats.move_mixer_to_live();
break;
},
Err(e) => {
loop_task = Some(e.0 .1);
let worker = self.workers.swap_remove(idx);
// NOTE: we have incremented worker's live counter for this mixer in
// `schedule_mixer`.
// The only time this branch is ever hit is if a worker crashed, so we
// need to replicate some of their cleanup.
self.stats
.remove_live_mixers(worker.stats().live_mixers().saturating_sub(1));
self.stats.remove_worker();
},
}
}
}
}
/// Fetch the first `Worker` that has room for a new task, creating one if needed.
///
/// If an inbound task has spilled from another thread, then do not reschedule it there.
fn fetch_worker(
&mut self,
task: &ParkedMixer,
avoid: Option<WorkerId>,
) -> (&mut Worker, usize) {
let idx = self
.workers
.iter()
.position(|w| w.can_schedule(task, avoid))
.unwrap_or_else(|| {
self.workers.push(Worker::new(
self.next_worker_id.incr(),
self.config.clone(),
self.tx.clone(),
self.stats.clone(),
));
self.stats.add_worker();
self.workers.len() - 1
});
(&mut self.workers[idx], idx)
}
pub fn spawn(mut self) {
tokio::spawn(async move { self.run().await });
}
}
#[cfg(test)]
mod test {
use super::*;
use crate::{
constants::test_data::FILE_WEBM_TARGET,
driver::{tasks::mixer::Mixer, OutputMode},
input::File,
Driver,
};
use tokio::runtime::Handle;
#[tokio::test]
async fn inactive_mixers_dont_need_threads() {
let sched = Scheduler::new(Config::default());
let cfg = DriverConfig::default().scheduler(sched.clone());
let _drivers: Vec<Driver> = (0..1024).map(|_| Driver::new(cfg.clone())).collect();
tokio::time::sleep(Duration::from_secs(1)).await;
assert_eq!(sched.total_tasks(), 1024);
assert_eq!(sched.live_tasks(), 0);
assert_eq!(sched.worker_threads(), 0);
}
#[tokio::test]
async fn active_mixers_spawn_threads() {
let config = Config {
strategy: Mode::default(),
move_expensive_tasks: false,
};
let sched = Scheduler::new(config);
let (pkt_tx, _pkt_rx) = flume::unbounded();
let cfg = DriverConfig::default()
.scheduler(sched.clone())
.override_connection(Some(OutputMode::Rtp(pkt_tx)));
let n_tasks = 1024;
let _drivers: Vec<Driver> = (0..n_tasks)
.map(|_| {
let mut driver = Driver::new(cfg.clone());
let file = File::new(FILE_WEBM_TARGET);
driver.play_input(file.into());
driver
})
.collect();
tokio::time::sleep(Duration::from_secs(10)).await;
assert_eq!(sched.total_tasks(), n_tasks);
assert_eq!(sched.live_tasks(), n_tasks);
assert_eq!(
sched.worker_threads(),
n_tasks / (DEFAULT_MIXERS_PER_THREAD.get() as u64)
);
}
#[tokio::test]
async fn excess_threads_are_cleaned_up() {
const TEST_TIMER: Duration = Duration::from_millis(500);
let config = Config {
strategy: Mode::MaxPerThread(1.try_into().unwrap()),
move_expensive_tasks: true,
};
let (mut core, tx) = Idle::new(config.clone());
core.cull_timer = TEST_TIMER;
let mut next_id = TaskId::new();
let mut thread_id = WorkerId::new();
let mut handles = vec![];
for i in 0..2 {
let mut worker = Worker::new(
thread_id.incr(),
config.clone(),
tx.clone(),
core.stats.clone(),
);
let ((mixer, listeners), track_handle) =
Mixer::test_with_float_unending(Handle::current(), false);
let send_mixer = ParkedMixer {
mixer: Box::new(mixer),
ssrc: i,
rtp_sequence: i as u16,
rtp_timestamp: i,
park_time: TokInstant::now().into(),
last_cost: None,
cull_handle: None,
};
core.stats.add_idle_mixer();
core.stats.move_mixer_to_live();
worker.schedule_mixer(next_id.incr(), send_mixer).unwrap();
handles.push((track_handle, listeners));
core.workers.push(worker);
}
let mut timer = tokio::time::interval(TIMESTEP_LENGTH);
assert!(core.run_once(&mut timer).await);
// Stop one of the handles, allow it to exit, and then run core again.
handles[1].0.stop().unwrap();
while core.workers[1].is_busy() {
assert!(core.run_once(&mut timer).await);
}
tokio::time::sleep(TEST_TIMER + Duration::from_secs(1)).await;
while core.workers.len() != 1 {
assert!(core.run_once(&mut timer).await);
}
assert_eq!(core.stats.worker_threads(), 0);
}
}

View File

@@ -0,0 +1,823 @@
use std::time::{Duration, Instant};
use discortp::rtp::{MutableRtpPacket, RtpPacket};
use flume::{SendError, TryRecvError};
use tokio::time::Instant as TokInstant;
use crate::{
constants::*,
driver::tasks::{error::Error as DriverError, mixer::Mixer},
};
#[cfg(test)]
use crate::driver::test_config::TickStyle;
use super::*;
/// The send-half of a worker thread, with bookkeeping mechanisms to help
/// the idle task schedule incoming tasks.
pub struct Worker {
id: WorkerId,
stats: Arc<LiveStatBlock>,
config: Config,
tx: Sender<(TaskId, ParkedMixer)>,
known_empty_since: Option<TokInstant>,
}
#[allow(missing_docs)]
impl Worker {
pub fn new(
id: WorkerId,
config: Config,
sched_tx: Sender<SchedulerMessage>,
global_stats: Arc<StatBlock>,
) -> Self {
let stats = Arc::new(LiveStatBlock::default());
let (live_tx, live_rx) = flume::unbounded();
let core = Live::new(
id,
config.clone(),
global_stats,
stats.clone(),
live_rx,
sched_tx,
);
core.spawn();
Self {
id,
stats,
config,
tx: live_tx,
known_empty_since: None,
}
}
/// Mark the worker thread as idle from the present time if it reports no tasks.
///
/// This time information is used for thread culling.
#[inline]
pub fn try_mark_empty(&mut self, now: TokInstant) -> Option<TokInstant> {
if self.stats.live_mixers() == 0 {
self.known_empty_since.get_or_insert(now);
} else {
self.mark_busy();
}
self.known_empty_since
}
/// Unset the thread culling time on this worker.
#[inline]
pub fn mark_busy(&mut self) {
self.known_empty_since = None;
}
#[cfg(test)]
#[inline]
pub fn is_busy(&mut self) -> bool {
self.known_empty_since.is_none()
}
/// Return whether this thread has enough room (task count, spare cycles)
/// for the given task.
#[inline]
#[must_use]
pub fn can_schedule(&self, task: &ParkedMixer, avoid: Option<WorkerId>) -> bool {
avoid.is_none_or(|id| !self.has_id(id)) && self.stats.has_room(&self.config.strategy, task)
}
#[inline]
#[must_use]
pub fn stats(&self) -> Arc<LiveStatBlock> {
self.stats.clone()
}
/// Increment this worker's statistics and hand off a task for execution.
#[inline]
pub fn schedule_mixer(
&mut self,
id: TaskId,
task: ParkedMixer,
) -> Result<(), SendError<(TaskId, ParkedMixer)>> {
self.mark_busy();
self.stats.add_mixer();
self.tx.send((id, task))
}
#[must_use]
pub fn has_id(&self, id: WorkerId) -> bool {
self.id == id
}
}
const PACKETS_PER_BLOCK: usize = 16;
const MEMORY_CULL_TIMER: Duration = Duration::from_secs(10);
/// A synchronous thread responsible for mixing, encoding, encrypting, and
/// sending the audio output of many `Mixer`s.
///
/// `Mixer`s remain `Box`ed due to large move costs, and unboxing them appeared to have
/// a 5--10% perf cost from benchmarks.
pub struct Live {
packets: Vec<Box<[u8]>>,
packet_lens: Vec<usize>,
#[allow(clippy::vec_box)]
tasks: Vec<Box<Mixer>>,
ids: Vec<TaskId>,
to_cull: Vec<bool>,
deadline: Instant,
start_of_work: Option<Instant>,
id: WorkerId,
config: Config,
stats: Arc<LiveStatBlock>,
global_stats: Arc<StatBlock>,
rx: Receiver<(TaskId, ParkedMixer)>,
tx: Sender<SchedulerMessage>,
excess_buffer_cull_time: Option<Instant>,
}
#[allow(missing_docs)]
impl Live {
pub fn new(
id: WorkerId,
config: Config,
global_stats: Arc<StatBlock>,
stats: Arc<LiveStatBlock>,
rx: Receiver<(TaskId, ParkedMixer)>,
tx: Sender<SchedulerMessage>,
) -> Self {
let to_prealloc = config.strategy.prealloc_size();
let block_size = config
.strategy
.task_limit()
.unwrap_or(PACKETS_PER_BLOCK)
.min(PACKETS_PER_BLOCK);
let packets = vec![packet_block(block_size)];
Self {
packets,
packet_lens: Vec::with_capacity(to_prealloc),
tasks: Vec::with_capacity(to_prealloc),
ids: Vec::with_capacity(to_prealloc),
to_cull: Vec::with_capacity(to_prealloc),
deadline: Instant::now(),
start_of_work: None,
id,
config,
stats,
global_stats,
rx,
tx,
excess_buffer_cull_time: None,
}
}
#[inline]
fn run(&mut self) {
while self.run_once() {}
self.global_stats.remove_worker();
}
/// Returns whether the loop should exit (i.e., culled by main `Scheduler`).
#[inline]
pub fn run_once(&mut self) -> bool {
// Check for new tasks.
if self.handle_scheduler_msgs().is_err() {
return false;
}
// Receive commands for each task.
self.handle_task_msgs();
// Move any idle calls back to the global pool.
self.demote_and_remove_mixers();
// Take a clock measure before and after each packet.
let mut pre_pkt_time = Instant::now();
let mut worst_task = (0, Duration::default());
for (i, (packet_len, mixer)) in self
.packet_lens
.iter_mut()
.zip(self.tasks.iter_mut())
.enumerate()
{
let (block, inner) = get_memory_indices(i);
match mixer.mix_and_build_packet(&mut self.packets[block][inner..][..VOICE_PACKET_MAX])
{
Ok(written_sz) => *packet_len = written_sz,
e => {
*packet_len = 0;
rebuild_if_err(mixer, e, &mut self.to_cull, i);
},
}
let post_pkt_time = Instant::now();
let cost = post_pkt_time.duration_since(pre_pkt_time);
if cost > worst_task.1 {
worst_task = (i, cost);
}
pre_pkt_time = post_pkt_time;
}
let end_of_work = pre_pkt_time;
if let Some(start_of_work) = self.start_of_work {
let ns_cost = self.stats.store_compute_cost(end_of_work - start_of_work);
if self.config.move_expensive_tasks
&& ns_cost >= RESCHEDULE_THRESHOLD
&& self.ids.len() > 1
{
self.offload_mixer(worst_task.0, worst_task.1);
}
}
self.timed_remove_excess_blocks(end_of_work);
// Wait till the right time to send this packet:
// usually a 20ms tick, in test modes this is either a finite number of runs or user input.
self.march_deadline();
// Send all.
self.start_of_work = Some(Instant::now());
for (i, (packet_len, mixer)) in self
.packet_lens
.iter_mut()
.zip(self.tasks.iter_mut())
.enumerate()
{
let (block, inner) = get_memory_indices(i);
let packet = &mut self.packets[block][inner..];
if *packet_len > 0 {
let res = mixer.send_packet(&packet[..*packet_len]);
rebuild_if_err(mixer, res, &mut self.to_cull, i);
}
#[cfg(test)]
if *packet_len == 0 {
mixer.test_signal_empty_tick();
}
advance_rtp_counters(packet);
}
for (i, mixer) in self.tasks.iter_mut().enumerate() {
let res = mixer
.audio_commands_events()
.and_then(|()| mixer.check_and_send_keepalive(self.start_of_work));
rebuild_if_err(mixer, res, &mut self.to_cull, i);
}
true
}
#[cfg(test)]
fn _march_deadline(&mut self) {
// For testing, assume all will have same tick style.
// Only count 'remaining loops' on one of the nodes.
let mixer = self.tasks.get_mut(0).map(|m| {
let style = m.config.tick_style.clone();
(m, style)
});
match mixer {
None | Some((_, TickStyle::Timed)) => {
std::thread::sleep(self.deadline.saturating_duration_since(Instant::now()));
self.deadline += TIMESTEP_LENGTH;
},
Some((m, TickStyle::UntimedWithExecLimit(rx))) => {
if m.remaining_loops.is_none() {
if let Ok(new_val) = rx.recv() {
m.remaining_loops = Some(new_val.wrapping_sub(1));
}
}
if let Some(cnt) = m.remaining_loops.as_mut() {
if *cnt == 0 {
m.remaining_loops = None;
} else {
*cnt = cnt.wrapping_sub(1);
}
}
},
}
}
#[cfg(not(test))]
#[inline(always)]
#[allow(clippy::inline_always)]
fn _march_deadline(&mut self) {
std::thread::sleep(self.deadline.saturating_duration_since(Instant::now()));
self.deadline += TIMESTEP_LENGTH;
}
#[inline]
fn march_deadline(&mut self) {
#[cfg(feature = "internals")]
{
return;
}
#[allow(clippy::used_underscore_items)]
self._march_deadline();
}
#[inline]
fn handle_scheduler_msgs(&mut self) -> Result<(), ()> {
let mut activation_time = None;
loop {
match self.rx.try_recv() {
Ok((id, task)) => {
self.add_task(
task,
id,
*activation_time.get_or_insert_with(|| {
self.deadline
.checked_sub(TIMESTEP_LENGTH)
.unwrap_or(self.deadline)
}),
);
},
Err(TryRecvError::Empty) => break,
Err(TryRecvError::Disconnected) => return Err(()),
}
}
Ok(())
}
/// Handle messages from each tasks's `Driver`, marking dead tasks for removal.
#[inline]
fn handle_task_msgs(&mut self) {
for (i, (packet, mixer)) in self
.packets
.iter_mut()
.flat_map(|v| v.chunks_exact_mut(VOICE_PACKET_MAX))
.zip(self.tasks.iter_mut())
.enumerate()
{
let mut events_failure = false;
let mut conn_failure = false;
let fatal = loop {
match mixer.mix_rx.try_recv() {
Ok(m) => {
let (events, conn, should_exit) = mixer.handle_message(m, packet);
events_failure |= events;
conn_failure |= conn;
if should_exit {
break true;
}
},
Err(TryRecvError::Disconnected) => {
break true;
},
Err(TryRecvError::Empty) => {
break false;
},
}
};
if fatal || mixer.do_rebuilds(events_failure, conn_failure).is_err() {
// this is not zipped in because it is *not* needed most ticks.
self.to_cull[i] = true;
}
}
}
#[cfg(feature = "internals")]
#[inline]
pub fn mark_for_cull(&mut self, idx: usize) {
self.to_cull[idx] = true;
}
/// Check and demote for any tasks without live audio sources who have sent all
/// necessary silent frames (or remove dead tasks).
///
/// This must occur *after* handling per-track events to prevent erroneously
/// descheduling tasks.
#[inline]
pub fn demote_and_remove_mixers(&mut self) {
let mut i = 0;
while i < self.tasks.len() {
#[cfg(test)]
let force_conn = self.tasks[i].config.override_connection.is_some();
#[cfg(not(test))]
let force_conn = false;
// Benchmarking suggests that these asserts remove some bounds checks for us.
assert!(i < self.tasks.len());
assert!(i < self.to_cull.len());
if self.to_cull[i]
|| (self.tasks[i].tracks.is_empty() && self.tasks[i].silence_frames == 0)
|| !(self.tasks[i].conn_active.is_some() || force_conn)
{
self.stats.remove_mixer();
if let Some((id, parked)) = self.remove_task(i) {
self.global_stats.move_mixer_to_idle();
_ = self.tx.send(SchedulerMessage::Demote(id, parked));
} else {
self.global_stats.remove_live_mixer();
}
} else {
i += 1;
}
}
}
/// Return a given mixer to the main scheduler if this worker is overloaded.
#[inline]
pub fn offload_mixer(&mut self, idx: usize, cost: Duration) {
self.stats.remove_mixer();
if let Some((id, mut parked)) = self.remove_task(idx) {
self.global_stats.move_mixer_to_idle();
parked.last_cost = Some(cost);
_ = self
.tx
.send(SchedulerMessage::Overspill(self.id, id, parked));
} else {
self.global_stats.remove_live_mixer();
}
}
#[inline]
fn needed_blocks(&self) -> usize {
let div = self.ids.len() / PACKETS_PER_BLOCK;
let rem = self.ids.len() % PACKETS_PER_BLOCK;
(rem != 0) as usize + div
}
#[inline]
fn has_excess_blocks(&self) -> bool {
self.packets.len() > self.needed_blocks()
}
#[inline]
fn remove_excess_blocks(&mut self) {
self.packets.truncate(self.needed_blocks());
}
/// Try to offload excess packet buffers.
///
/// If there is currently overallocation, then store the first time at which
/// this was seenb. If this condition persists past `MEMORY_CULL_TIMER`, remove
/// unnecessary blocks.
#[inline]
fn timed_remove_excess_blocks(&mut self, now: Instant) {
if self.has_excess_blocks() {
if let Some(mark_time) = self.excess_buffer_cull_time {
if now.duration_since(mark_time) >= MEMORY_CULL_TIMER {
self.remove_excess_blocks();
self.excess_buffer_cull_time = None;
}
} else {
self.excess_buffer_cull_time = Some(now);
}
} else {
self.excess_buffer_cull_time = None;
}
}
#[inline]
fn add_task(&mut self, task: ParkedMixer, id: TaskId, activation_time: Instant) {
let idx = self.ids.len();
let elapsed = task.park_time - activation_time;
let samples_f64 = elapsed.as_secs_f64() * (SAMPLE_RATE_RAW as f64);
let mod_samples = (samples_f64 as u64) as u32;
let rtp_timestamp = task.rtp_timestamp.wrapping_add(mod_samples);
self.ids.push(id);
self.tasks.push(task.mixer);
self.packet_lens.push(0);
self.to_cull.push(false);
let (block, inner_idx) = get_memory_indices(idx);
while self.packets.len() <= block {
self.add_packet_block();
}
let packet = &mut self.packets[block][inner_idx..][..VOICE_PACKET_MAX];
let mut rtp = MutableRtpPacket::new(packet).expect(
"FATAL: Too few bytes in self.packet for RTP header.\
(Blame: VOICE_PACKET_MAX?)",
);
rtp.set_ssrc(task.ssrc);
rtp.set_timestamp(rtp_timestamp.into());
rtp.set_sequence(task.rtp_sequence.into());
}
/// Allocate and store a new packet block.
///
/// This will be full-size (`PACKETS_PER_BLOCK`) unless this block
/// is a) the last required for the task limit and b) this limit
/// is not aligned to `PACKETS_PER_BLOCK`.
#[inline]
fn add_packet_block(&mut self) {
let n_packets = if let Some(limit) = self.config.strategy.task_limit() {
let (block, inner) = get_memory_indices_unscaled(limit);
if self.packets.len() < block || inner == 0 {
PACKETS_PER_BLOCK
} else {
inner
}
} else {
PACKETS_PER_BLOCK
};
self.packets.push(packet_block(n_packets));
}
#[cfg(any(test, feature = "internals"))]
#[inline]
pub fn add_task_direct(&mut self, task: Mixer, id: TaskId) {
let id_0 = id.get();
self.add_task(
ParkedMixer {
mixer: Box::new(task),
ssrc: id_0 as u32,
rtp_sequence: id_0 as u16,
rtp_timestamp: id_0 as u32,
park_time: Instant::now(),
last_cost: None,
cull_handle: None,
},
id,
Instant::now(),
);
}
/// Remove a `Mixer`, returning it to the idle scheduler.
///
/// This operates by `swap_remove`ing each element of a Mixer's state, including
/// on RTP packet headers. This is achieved by setting up a memcpy between
/// buffer segments.
#[inline]
pub fn remove_task(&mut self, idx: usize) -> Option<(TaskId, ParkedMixer)> {
let end = self.tasks.len() - 1;
let id = self.ids.swap_remove(idx);
let _len = self.packet_lens.swap_remove(idx);
let mixer = self.tasks.swap_remove(idx);
let alive = !self.to_cull.swap_remove(idx);
let (block, inner_idx) = get_memory_indices(idx);
let (removed, replacement) = if end > idx {
let (end_block, end_inner) = get_memory_indices(end);
let (rest, target_block) = self.packets.split_at_mut(end_block);
let (last_block, end_pkt) = target_block[0].split_at_mut(end_inner);
if end_block == block {
(&mut last_block[inner_idx..], Some(end_pkt))
} else {
(&mut rest[block][inner_idx..], Some(end_pkt))
}
} else {
(&mut self.packets[block][inner_idx..], None)
};
let rtp = RtpPacket::new(removed).expect(
"FATAL: Too few bytes in self.packet for RTP header.\
(Blame: VOICE_PACKET_MAX?)",
);
let ssrc = rtp.get_ssrc();
let rtp_timestamp = rtp.get_timestamp().into();
let rtp_sequence = rtp.get_sequence().into();
if let Some(replacement) = replacement {
// Copy the whole packet header since we know it'll be 4B aligned.
// 'Just necessary fields' is 2B aligned.
const COPY_LEN: usize = RtpPacket::minimum_packet_size();
removed[..COPY_LEN].copy_from_slice(&replacement[..COPY_LEN]);
}
alive.then(move || {
let park_time = Instant::now();
(
id,
ParkedMixer {
mixer,
ssrc,
rtp_sequence,
rtp_timestamp,
park_time,
last_cost: None,
cull_handle: None,
},
)
})
}
/// Spawn a new sync thread to manage `Mixer`s.
fn spawn(mut self) {
std::thread::spawn(move || {
self.run();
});
}
}
/// Initialises a packet block of the required size, prefilling any constant RTP data.
#[inline]
fn packet_block(n_packets: usize) -> Box<[u8]> {
let mut packets = vec![0u8; VOICE_PACKET_MAX * n_packets].into_boxed_slice();
for packet in packets.chunks_exact_mut(VOICE_PACKET_MAX) {
let mut rtp = MutableRtpPacket::new(packet).expect(
"FATAL: Too few bytes in self.packet for RTP header.\
(Blame: VOICE_PACKET_MAX?)",
);
rtp.set_version(RTP_VERSION);
rtp.set_payload_type(RTP_PROFILE_TYPE);
}
packets
}
/// Returns the block index into `self.packets` and the packet number in
/// the block for a given worker's index.
#[inline]
fn get_memory_indices_unscaled(idx: usize) -> (usize, usize) {
let block_size = PACKETS_PER_BLOCK;
(idx / block_size, idx % block_size)
}
/// Returns the block index into `self.packets` and the byte offset into
/// a packet block for a given worker's index.
#[inline]
fn get_memory_indices(idx: usize) -> (usize, usize) {
let (block, inner_unscaled) = get_memory_indices_unscaled(idx);
(block, inner_unscaled * VOICE_PACKET_MAX)
}
#[inline]
fn advance_rtp_counters(packet: &mut [u8]) {
let mut rtp = MutableRtpPacket::new(packet).expect(
"FATAL: Too few bytes in self.packet for RTP header.\
(Blame: VOICE_PACKET_MAX?)",
);
rtp.set_sequence(rtp.get_sequence() + 1);
rtp.set_timestamp(rtp.get_timestamp() + MONO_FRAME_SIZE as u32);
}
/// Structured slightly confusingly: we only want to even access `cull_markers`
/// in the event of error.
#[inline]
fn rebuild_if_err<T>(
mixer: &mut Box<Mixer>,
res: Result<T, DriverError>,
cull_markers: &mut [bool],
idx: usize,
) {
if let Err(e) = res {
cull_markers[idx] |= mixer
.do_rebuilds(
e.should_trigger_interconnect_rebuild(),
e.should_trigger_connect(),
)
.is_err();
}
}
#[cfg(test)]
mod test {
use super::*;
use crate::driver::test_impls::*;
use tokio::runtime::Handle;
fn rtp_has_index(pkt: &[u8], sentinel_val: u16) {
let rtp = RtpPacket::new(pkt).unwrap();
assert_eq!(rtp.get_version(), RTP_VERSION);
assert_eq!(rtp.get_padding(), 0);
assert_eq!(rtp.get_extension(), 0);
assert_eq!(rtp.get_csrc_count(), 0);
assert_eq!(rtp.get_marker(), 0);
assert_eq!(rtp.get_payload_type(), RTP_PROFILE_TYPE);
assert_eq!(rtp.get_sequence(), sentinel_val.into());
assert_eq!(rtp.get_timestamp(), (sentinel_val as u32).into());
assert_eq!(rtp.get_ssrc(), sentinel_val as u32);
}
#[tokio::test]
async fn block_alloc_is_partial_small() {
let n_mixers = 1;
let (sched, _listeners) = MockScheduler::from_mixers(
Some(Mode::MaxPerThread(n_mixers.try_into().unwrap())),
(0..n_mixers)
.map(|_| Mixer::test_with_float(1, Handle::current(), false))
.collect(),
);
assert_eq!(sched.core.packets.len(), 1);
assert_eq!(sched.core.packets[0].len(), VOICE_PACKET_MAX);
}
#[tokio::test]
async fn block_alloc_is_partial_large() {
let n_mixers = 33;
let (sched, _listeners) = MockScheduler::from_mixers(
Some(Mode::MaxPerThread(n_mixers.try_into().unwrap())),
(0..n_mixers)
.map(|_| Mixer::test_with_float(1, Handle::current(), false))
.collect(),
);
assert_eq!(sched.core.packets.len(), 3);
assert_eq!(
sched.core.packets[0].len(),
PACKETS_PER_BLOCK * VOICE_PACKET_MAX
);
assert_eq!(
sched.core.packets[1].len(),
PACKETS_PER_BLOCK * VOICE_PACKET_MAX
);
assert_eq!(sched.core.packets[2].len(), VOICE_PACKET_MAX);
}
#[tokio::test]
async fn deletion_moves_pkt_header() {
let (mut sched, _listeners) = MockScheduler::from_mixers(
None,
(0..PACKETS_PER_BLOCK)
.map(|_| Mixer::test_with_float(1, Handle::current(), false))
.collect(),
);
let last_idx = (PACKETS_PER_BLOCK - 1) as u16;
// Remove head.
sched.core.remove_task(0);
rtp_has_index(&sched.core.packets[0], last_idx);
// Remove head.
sched.core.remove_task(5);
rtp_has_index(&sched.core.packets[0][5 * VOICE_PACKET_MAX..], last_idx - 1);
}
#[tokio::test]
async fn deletion_moves_pkt_header_multiblock() {
let n_pkts = PACKETS_PER_BLOCK + 8;
let (mut sched, _listeners) = MockScheduler::from_mixers(
None,
(0..n_pkts)
.map(|_| Mixer::test_with_float(1, Handle::current(), false))
.collect(),
);
let last_idx = (n_pkts - 1) as u16;
// Remove head (read from block 1 into block 0).
sched.core.remove_task(0);
rtp_has_index(&sched.core.packets[0], last_idx);
// Remove later (read from block 1 into block 1).
sched.core.remove_task(17);
rtp_has_index(&sched.core.packets[1][VOICE_PACKET_MAX..], last_idx - 1);
}
#[tokio::test]
async fn packet_blocks_are_cleaned_up() {
// Allocate 2 blocks.
let n_pkts = PACKETS_PER_BLOCK + 1;
let (mut sched, _listeners) = MockScheduler::from_mixers(
None,
(0..n_pkts)
.map(|_| Mixer::test_with_float(1, Handle::current(), false))
.collect(),
);
// Assert no cleanup at start.
assert!(sched.core.run_once());
assert_eq!(sched.core.needed_blocks(), 2);
assert!(sched.core.excess_buffer_cull_time.is_none());
// Remove only entry in last block. Cleanup should be sched'd.
sched.core.remove_task(n_pkts - 1);
assert!(sched.core.run_once());
assert!(sched.core.has_excess_blocks());
assert!(sched.core.excess_buffer_cull_time.is_some());
tokio::time::sleep(Duration::from_secs(2) + MEMORY_CULL_TIMER).await;
// Cleanup should be unsched'd.
assert!(sched.core.run_once());
assert!(sched.core.excess_buffer_cull_time.is_none());
assert!(!sched.core.has_excess_blocks());
}
}

View File

@@ -0,0 +1,180 @@
use std::{
error::Error as StdError,
fmt::Display,
num::NonZeroUsize,
sync::{Arc, OnceLock},
};
use flume::{Receiver, RecvError, Sender};
use super::tasks::message::{Interconnect, MixerMessage};
use crate::{constants::TIMESTEP_LENGTH, Config as DriverConfig};
mod config;
mod idle;
mod live;
mod stats;
mod task;
pub use config::*;
use idle::*;
pub use live::*;
pub use stats::*;
pub use task::*;
/// A soft maximum of 90% of the 20ms budget to account for variance in execution time.
const RESCHEDULE_THRESHOLD: u64 = ((TIMESTEP_LENGTH.subsec_nanos() as u64) * 9) / 10;
const DEFAULT_MIXERS_PER_THREAD: NonZeroUsize = match NonZeroUsize::new(16) {
Some(v) => v,
None => unreachable!(),
};
/// The default shared scheduler instance.
///
/// This is built using the default value of [`ScheduleMode`]. Users desiring
/// a custom strategy should avoid calling [`Config::default`].
///
/// [`Config::default`]: crate::Config::default
/// [`ScheduleMode`]: Mode
pub fn get_default_scheduler() -> &'static Scheduler {
static DEFAULT_SCHEDULER: OnceLock<Scheduler> = OnceLock::new();
DEFAULT_SCHEDULER.get_or_init(Scheduler::default)
}
/// A reference to a shared group of threads used for running idle and active
/// audio threads.
#[derive(Clone, Debug)]
pub struct Scheduler {
inner: Arc<InnerScheduler>,
}
/// Inner contents of a [`Scheduler`] instance.
///
/// This is an `Arc` around `Arc`'d contents so that we can make use of the
/// drop check on `Scheduler` to clean up resources.
#[derive(Clone, Debug)]
struct InnerScheduler {
tx: Sender<SchedulerMessage>,
stats: Arc<StatBlock>,
}
impl Scheduler {
/// Create a new mixer scheduler from the allocation strategy in `config`.
#[must_use]
pub fn new(config: Config) -> Self {
let (core, tx) = Idle::new(config);
let stats = core.stats.clone();
core.spawn();
let inner = Arc::new(InnerScheduler { tx, stats });
Self { inner }
}
pub(crate) fn new_mixer(
&self,
config: &DriverConfig,
ic: Interconnect,
rx: Receiver<MixerMessage>,
) {
self.inner
.tx
.send(SchedulerMessage::NewMixer(rx, ic, config.clone()))
.unwrap();
}
/// Returns the total number of calls (idle and active) scheduled.
#[must_use]
pub fn total_tasks(&self) -> u64 {
self.inner.stats.total_mixers()
}
/// Returns the total number of *active* calls scheduled and processing
/// audio.
#[must_use]
pub fn live_tasks(&self) -> u64 {
self.inner.stats.live_mixers()
}
/// Returns the total number of threads spawned to process live audio sessions.
#[must_use]
pub fn worker_threads(&self) -> u64 {
self.inner.stats.worker_threads()
}
/// Request a list of handles to statistics for currently live workers.
pub async fn worker_thread_stats(&self) -> Result<Vec<Arc<LiveStatBlock>>, Error> {
let (tx, rx) = flume::bounded(1);
_ = self.inner.tx.send(SchedulerMessage::GetStats(tx));
rx.recv_async().await.map_err(Error::from)
}
/// Request a list of handles to statistics for currently live workers with a blocking call.
pub fn worker_thread_stats_blocking(&self) -> Result<Vec<Arc<LiveStatBlock>>, Error> {
let (tx, rx) = flume::bounded(1);
_ = self.inner.tx.send(SchedulerMessage::GetStats(tx));
rx.recv().map_err(Error::from)
}
}
impl Drop for InnerScheduler {
fn drop(&mut self) {
_ = self.tx.send(SchedulerMessage::Kill);
}
}
impl Default for Scheduler {
fn default() -> Self {
Scheduler::new(Config::default())
}
}
/// Control messages for a scheduler.
pub enum SchedulerMessage {
/// Build a new `Mixer` as part of the initialisation of a `Driver`.
NewMixer(Receiver<MixerMessage>, Interconnect, DriverConfig),
/// Forward a command for
Do(TaskId, MixerMessage),
/// Return a `Mixer` from a worker back to the idle pool.
Demote(TaskId, ParkedMixer),
/// Move an expensive `Mixer` to another thread in the worker pool.
Overspill(WorkerId, TaskId, ParkedMixer),
/// Request a copy of all per-worker statistics.
GetStats(Sender<Vec<Arc<LiveStatBlock>>>),
/// Cleanup once all `Scheduler` handles are dropped.
Kill,
}
/// Errors encountered when communicating with the internals of a [`Scheduler`].
///
/// [`Scheduler`]: crate::driver::Scheduler
#[non_exhaustive]
#[derive(Debug)]
pub enum Error {
/// The scheduler exited or crashed while awating the request.
Disconnected,
}
impl Display for Error {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Disconnected => f.write_str("the scheduler terminated mid-request"),
}
}
}
impl StdError for Error {
fn source(&self) -> Option<&(dyn StdError + 'static)> {
None
}
}
impl From<RecvError> for Error {
fn from(_: RecvError) -> Self {
Self::Disconnected
}
}

View File

@@ -0,0 +1,134 @@
use std::{
sync::atomic::{AtomicU64, Ordering},
time::Duration,
};
use super::{Mode, ParkedMixer, RESCHEDULE_THRESHOLD};
/// Statistics shared by an entire `Scheduler`.
#[derive(Debug, Default)]
pub struct StatBlock {
total: AtomicU64,
live: AtomicU64,
threads: AtomicU64,
}
#[allow(missing_docs)]
impl StatBlock {
#[inline]
pub fn total_mixers(&self) -> u64 {
self.total.load(Ordering::Relaxed)
}
#[inline]
pub fn live_mixers(&self) -> u64 {
self.live.load(Ordering::Relaxed)
}
#[inline]
pub fn worker_threads(&self) -> u64 {
self.threads.load(Ordering::Relaxed)
}
#[inline]
pub fn add_idle_mixer(&self) {
self.total.fetch_add(1, Ordering::Relaxed);
}
#[inline]
pub fn remove_idle_mixers(&self, n: u64) {
self.total.fetch_sub(n, Ordering::Relaxed);
}
#[inline]
pub fn move_mixer_to_live(&self) {
self.live.fetch_add(1, Ordering::Relaxed);
}
#[inline]
pub fn move_mixer_to_idle(&self) {
self.move_mixers_to_idle(1);
}
#[inline]
pub fn move_mixers_to_idle(&self, n: u64) {
self.live.fetch_sub(n, Ordering::Relaxed);
}
#[inline]
pub fn remove_live_mixer(&self) {
self.remove_live_mixers(1);
}
#[inline]
pub fn remove_live_mixers(&self, n: u64) {
self.move_mixers_to_idle(n);
self.remove_idle_mixers(n);
}
#[inline]
pub fn add_worker(&self) {
self.threads.fetch_add(1, Ordering::Relaxed);
}
#[inline]
pub fn remove_worker(&self) {
self.threads.fetch_sub(1, Ordering::Relaxed);
}
}
/// Runtime statistics for an individual worker.
///
/// Individual statistics are measured atomically -- the worker thread
/// may have been cleaned up, or its mixer count may not match the
/// count when [`Self::last_compute_cost_ns`] was set.
#[derive(Debug, Default)]
pub struct LiveStatBlock {
live: AtomicU64,
last_ns: AtomicU64,
}
impl LiveStatBlock {
/// Returns the number of mixer tasks scheduled on this worker thread.
#[inline]
pub fn live_mixers(&self) -> u64 {
self.live.load(Ordering::Relaxed)
}
#[inline]
pub(crate) fn add_mixer(&self) {
self.live.fetch_add(1, Ordering::Relaxed);
}
#[inline]
pub(crate) fn remove_mixer(&self) {
self.live.fetch_sub(1, Ordering::Relaxed);
}
#[inline]
pub(crate) fn store_compute_cost(&self, work: Duration) -> u64 {
let cost = work.as_nanos() as u64;
self.last_ns.store(cost, Ordering::Relaxed);
cost
}
/// Returns the number of nanoseconds required to process all worker threads'
/// packet transmission, mixing, encoding, and encryption in the last tick.
#[inline]
pub fn last_compute_cost_ns(&self) -> u64 {
self.last_ns.load(Ordering::Relaxed)
}
#[inline]
pub(crate) fn has_room(&self, strategy: &Mode, task: &ParkedMixer) -> bool {
let task_room = strategy
.task_limit()
.is_none_or(|limit| self.live_mixers() < limit as u64);
let exec_room = task.last_cost.is_none_or(|cost| {
cost.as_nanos() as u64 + self.last_compute_cost_ns() < RESCHEDULE_THRESHOLD
});
task_room && exec_room
}
}

View File

@@ -0,0 +1,214 @@
use std::{
marker::PhantomData,
time::{Duration, Instant},
};
use flume::{Receiver, Sender};
use nohash_hasher::IsEnabled;
use rand::random;
use tokio::runtime::Handle;
use crate::{
driver::tasks::{
error::Error as DriverError,
message::{EventMessage, Interconnect, MixerMessage},
mixer::Mixer,
},
Config,
};
use super::SchedulerMessage;
/// Typesafe counter used to identify individual mixer/worker instances.
#[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct ResId<T>(u64, PhantomData<T>);
#[allow(missing_docs)]
pub type TaskId = ResId<TaskMarker>;
#[allow(missing_docs)]
pub type WorkerId = ResId<WorkerMarker>;
#[allow(missing_docs)]
#[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct TaskMarker;
#[allow(missing_docs)]
#[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct WorkerMarker;
impl<T> IsEnabled for ResId<T> {}
#[allow(missing_docs)]
impl<T: Copy> ResId<T> {
#[must_use]
pub fn new() -> Self {
Self::default()
}
pub fn incr(&mut self) -> Self {
let out = *self;
self.0 = self.0.wrapping_add(1);
out
}
#[cfg(any(test, feature = "internals"))]
#[must_use]
pub fn get(self) -> u64 {
self.0
}
}
impl<T: Copy> Default for ResId<T> {
fn default() -> Self {
Self(0, PhantomData)
}
}
/// An idle mixer instance, externally controlled by a `Driver`.
///
/// Since we do not allocate packet buffers for idle threads, this
/// struct includes various RTP fields.
pub struct ParkedMixer {
/// Mixer, track, etc. state as well as message receivers.
pub mixer: Box<Mixer>,
/// The SSRC assigned to this voice session.
pub ssrc: u32,
/// The last recorded/generated RTP sequence.
pub rtp_sequence: u16,
/// The last recorded/generated RTP timestamp.
pub rtp_timestamp: u32,
/// The time at which this `Mixer` was made idle.
///
/// This is used when transitioning to a live state to determine
/// how far we should adjust the RTP timestamp by.
pub park_time: Instant,
/// The last known cost of executing this task, if it had to be moved
/// due to a limit on thread resources.
pub last_cost: Option<Duration>,
/// Handle to any forwarder task, used if this mixer is culled while idle.
pub cull_handle: Option<Sender<()>>,
}
#[allow(missing_docs)]
impl ParkedMixer {
/// Create a new `Mixer` in a parked state.
#[must_use]
pub fn new(mix_rx: Receiver<MixerMessage>, interconnect: Interconnect, config: Config) -> Self {
Self {
mixer: Box::new(Mixer::new(mix_rx, Handle::current(), interconnect, config)),
ssrc: 0,
rtp_sequence: random::<u16>(),
rtp_timestamp: random::<u32>(),
park_time: Instant::now(),
last_cost: None,
cull_handle: None,
}
}
/// Spawn a tokio task which forwards any mixer messages to the central `Idle` task pool.
///
/// Any requests which would cause this mixer to become live will terminate
/// this task.
pub fn spawn_forwarder(&mut self, tx: Sender<SchedulerMessage>, id: TaskId) {
let (kill_tx, kill_rx) = flume::bounded(1);
self.cull_handle = Some(kill_tx);
let remote_rx = self.mixer.mix_rx.clone();
tokio::spawn(async move {
loop {
tokio::select! {
biased;
_ = kill_rx.recv_async() => break,
msg = remote_rx.recv_async() => {
let exit = if let Ok(msg) = msg {
let remove_self = msg.is_mixer_maybe_live();
tx.send_async(SchedulerMessage::Do(id, msg)).await.is_err() || remove_self
} else {
true
};
if exit {
break;
}
}
}
}
});
}
/// Returns whether the mixer should exit and be cleaned up.
pub fn handle_message(&mut self, msg: MixerMessage) -> Result<bool, ()> {
match msg {
MixerMessage::SetConn(conn, ssrc) => {
// Overridden because payload-specific fields are carried
// externally on `ParkedMixer`.
self.ssrc = ssrc;
self.rtp_sequence = random::<u16>();
self.rtp_timestamp = random::<u32>();
self.mixer.conn_active = Some(conn);
self.mixer.update_keepalive(ssrc);
Ok(false)
},
MixerMessage::Ws(ws) => {
// Overridden so that we don't mistakenly tell Discord we're speaking.
self.mixer.ws = ws;
self.send_gateway_not_speaking();
Ok(false)
},
msg => {
let (events_failure, conn_failure, should_exit) =
self.mixer.handle_message(msg, &mut []);
self.mixer
.do_rebuilds(events_failure, conn_failure)
.map_err(|_| ())
.map(|()| should_exit)
},
}
}
/// Handle periodic events attached to this `Mixer`, including timer state
/// on the event thread and UDP keepalives needed to prevent session termination.
///
/// As we init our UDP sockets as non-blocking via Tokio -> `into_std`, it is
/// safe to call UDP packet sends like this.
pub fn tick_and_keepalive(&mut self, now: Instant) -> Result<(), ()> {
// TODO: should we include an atomic which signals whether the event
// thread *cares*, so we can prevent wakeups?
// Can we do the same for live tracks?
let mut events_failure = self.mixer.fire_event(EventMessage::Tick).is_err();
let ka_err = self
.mixer
.check_and_send_keepalive(Some(now))
.or_else(DriverError::disarm_would_block);
let conn_failure = if let Err(e) = ka_err {
events_failure |= e.should_trigger_interconnect_rebuild();
e.should_trigger_connect()
} else {
false
};
self.mixer
.do_rebuilds(events_failure, conn_failure)
.map_err(|_| ())
}
pub fn send_gateway_speaking(&mut self) -> Result<(), ()> {
if let Err(e) = self.mixer.send_gateway_speaking() {
self.mixer
.do_rebuilds(
e.should_trigger_interconnect_rebuild(),
e.should_trigger_connect(),
)
.map_err(|_| ())
} else {
Ok(())
}
}
pub fn send_gateway_not_speaking(&self) {
self.mixer.send_gateway_not_speaking();
}
}

View File

@@ -0,0 +1,44 @@
use super::message::*;
use flume::{Receiver, Sender};
use tracing::{instrument, trace};
#[derive(Debug, Clone)]
pub struct DisposalThread(Sender<DisposalMessage>);
impl Default for DisposalThread {
fn default() -> Self {
Self::run()
}
}
impl DisposalThread {
#[must_use]
pub fn run() -> Self {
let (mix_tx, mix_rx) = flume::unbounded();
std::thread::spawn(move || {
trace!("Disposal thread started.");
runner(mix_rx);
trace!("Disposal thread finished.");
});
Self(mix_tx)
}
pub(super) fn dispose(&self, message: DisposalMessage) {
drop(self.0.send(message));
}
}
/// The mixer's disposal thread is also synchronous, due to tracks,
/// inputs, etc. being based on synchronous I/O.
///
/// The mixer uses this to offload heavy and expensive drop operations
/// to prevent deadline misses.
#[instrument(skip(mix_rx))]
#[expect(
clippy::needless_pass_by_value,
reason = "spawned on background thread, must take by value"
)]
fn runner(mix_rx: Receiver<DisposalMessage>) {
while mix_rx.recv().is_ok() {}
}

View File

@@ -0,0 +1,150 @@
use super::message::*;
use crate::ws::Error as WsError;
use aes_gcm::Error as CryptoError;
use davey::errors::EncryptError as DaveyEncryptError;
use flume::SendError;
use opus2::Error as OpusError;
use std::io::{Error as IoError, ErrorKind as IoErrorKind};
#[derive(Debug)]
pub enum Recipient {
AuxNetwork,
Event,
Mixer,
#[cfg(feature = "receive")]
UdpRx,
}
pub type Result<T> = std::result::Result<T, Error>;
#[derive(Debug)]
#[non_exhaustive]
pub enum Error {
Crypto(CryptoError),
DaveEncrypt,
#[cfg(any(feature = "receive", test))]
/// Received an illegal voice packet on the voice UDP socket.
IllegalVoicePacket,
InterconnectFailure(Recipient),
Io(IoError),
Other,
}
impl Error {
pub(crate) fn should_trigger_connect(&self) -> bool {
match self {
Error::InterconnectFailure(Recipient::AuxNetwork) => true,
#[cfg(feature = "receive")]
Error::InterconnectFailure(Recipient::UdpRx) => true,
_ => false,
}
}
pub(crate) fn should_trigger_interconnect_rebuild(&self) -> bool {
matches!(self, Error::InterconnectFailure(Recipient::Event))
}
// This prevents a `WouldBlock` from triggering a full reconnect,
// instead simply dropping the packet.
pub(crate) fn disarm_would_block(self) -> Result<()> {
match self {
Self::Io(i) if i.kind() == IoErrorKind::WouldBlock => Ok(()),
e => Err(e),
}
}
}
impl From<CryptoError> for Error {
fn from(e: CryptoError) -> Self {
Error::Crypto(e)
}
}
impl From<DaveyEncryptError> for Error {
fn from(_e: DaveyEncryptError) -> Self {
Self::DaveEncrypt
}
}
impl From<IoError> for Error {
fn from(e: IoError) -> Error {
Error::Io(e)
}
}
impl From<OpusError> for Error {
fn from(_: OpusError) -> Error {
Error::Other
}
}
impl From<SendError<WsMessage>> for Error {
fn from(_e: SendError<WsMessage>) -> Error {
Error::InterconnectFailure(Recipient::AuxNetwork)
}
}
impl From<SendError<EventMessage>> for Error {
fn from(_e: SendError<EventMessage>) -> Error {
Error::InterconnectFailure(Recipient::Event)
}
}
impl From<SendError<MixerMessage>> for Error {
fn from(_e: SendError<MixerMessage>) -> Error {
Error::InterconnectFailure(Recipient::Mixer)
}
}
#[cfg(feature = "receive")]
impl From<SendError<UdpRxMessage>> for Error {
fn from(_e: SendError<UdpRxMessage>) -> Error {
Error::InterconnectFailure(Recipient::UdpRx)
}
}
impl From<WsError> for Error {
fn from(_: WsError) -> Error {
Error::Other
}
}
#[derive(Debug)]
#[non_exhaustive]
pub enum DaveReinitError {
Init,
Reinit,
Reset,
CreateKeyPackage,
Ws(WsError),
}
impl From<davey::errors::InitError> for DaveReinitError {
fn from(_e: davey::errors::InitError) -> Self {
Self::Init
}
}
impl From<davey::errors::ReinitError> for DaveReinitError {
fn from(_e: davey::errors::ReinitError) -> Self {
Self::Reinit
}
}
impl From<davey::errors::ResetError> for DaveReinitError {
fn from(_e: davey::errors::ResetError) -> Self {
Self::Reset
}
}
impl From<davey::errors::CreateKeyPackageError> for DaveReinitError {
fn from(_e: davey::errors::CreateKeyPackageError) -> Self {
Self::CreateKeyPackage
}
}
impl From<WsError> for DaveReinitError {
fn from(e: WsError) -> Self {
Self::Ws(e)
}
}

View File

@@ -0,0 +1,139 @@
use super::message::*;
use crate::{
events::{EventStore, GlobalEvents, TrackEvent},
tracks::{ReadyState, TrackHandle, TrackState},
};
use flume::Receiver;
use tracing::{debug, info, instrument, trace};
#[instrument(skip(evt_rx))]
pub(crate) async fn runner(evt_rx: Receiver<EventMessage>) {
let mut global = GlobalEvents::default();
let mut events: Vec<EventStore> = vec![];
let mut states: Vec<TrackState> = vec![];
let mut handles: Vec<TrackHandle> = vec![];
while let Ok(msg) = evt_rx.recv_async().await {
match msg {
EventMessage::AddGlobalEvent(data) => {
info!("Global event added.");
global.add_event(data);
},
EventMessage::AddTrackEvent(i, data) => {
info!("Adding event to track {}.", i);
let event_store = events
.get_mut(i)
.expect("Event thread was given an illegal store index for AddTrackEvent.");
let state = states
.get_mut(i)
.expect("Event thread was given an illegal state index for AddTrackEvent.");
event_store.add_event(data, state.position);
},
EventMessage::FireCoreEvent(ctx) => {
let ctx = ctx.to_user_context();
let evt = ctx
.to_core_event()
.expect("Event thread was passed a non-core event in FireCoreEvent.");
trace!("Firing core event {:?}.", evt);
global.fire_core_event(evt, ctx).await;
},
EventMessage::RemoveGlobalEvents => {
global.remove_handlers();
},
EventMessage::AddTrack(store, state, handle) => {
events.push(store);
states.push(state);
handles.push(handle);
info!("Event state for track {} added", events.len());
},
EventMessage::ChangeState(i, change) => {
let max_states = states.len();
debug!(
"Changing state for track {} of {}: {:?}",
i, max_states, change
);
let state = states
.get_mut(i)
.expect("Event thread was given an illegal state index for ChangeState.");
match change {
TrackStateChange::Mode(mut mode) => {
std::mem::swap(&mut state.playing, &mut mode);
if state.playing != mode {
global.fire_track_event(state.playing.as_track_event(), i);
if let Some(other_evts) = state.playing.also_fired_track_events() {
for evt in other_evts {
global.fire_track_event(evt, i);
}
}
}
},
TrackStateChange::Volume(vol) => {
state.volume = vol;
},
TrackStateChange::Position(pos) => {
// Currently, only Tick should fire time events.
state.position = pos;
},
TrackStateChange::Loops(loops, user_set) => {
state.loops = loops;
if !user_set {
global.fire_track_event(TrackEvent::Loop, i);
}
},
TrackStateChange::Total(new) => {
// Massive, unprecedented state changes.
*state = new;
},
TrackStateChange::Ready(ready_state) => {
state.ready = ready_state;
match ready_state {
ReadyState::Playable => {
global.fire_track_event(TrackEvent::Playable, i);
},
ReadyState::Preparing => {
global.fire_track_event(TrackEvent::Preparing, i);
},
ReadyState::Uninitialised => {},
}
},
}
},
EventMessage::RemoveAllTracks => {
info!("Event state for all tracks removed.");
events.clear();
states.clear();
handles.clear();
},
EventMessage::Tick => {
// NOTE: this should fire saved up blocks of state change evts.
global.tick(&mut events, &mut states, &mut handles).await;
let mut i = 0;
while i < states.len() {
if states[i].playing.is_done() {
info!("Event state for track {} of {} removed.", i, events.len());
events.swap_remove(i);
states.swap_remove(i);
handles.swap_remove(i);
} else {
i += 1;
}
}
},
EventMessage::Poison => break,
}
}
trace!("Event thread exited.");
}

View File

@@ -0,0 +1,33 @@
#![allow(missing_docs)]
use crate::{
driver::{connection::error::Error, Bitrate, Config},
events::{context_data::DisconnectReason, EventData},
tracks::{Track, TrackCommand, TrackHandle},
ConnectionInfo,
};
use flume::{Receiver, Sender};
pub enum CoreMessage {
ConnectWithResult(ConnectionInfo, Sender<Result<(), Error>>),
RetryConnect(usize),
SignalWsClosure(usize, ConnectionInfo, Option<DisconnectReason>),
Disconnect,
SetTrack(Option<Box<TrackContext>>),
AddTrack(Box<TrackContext>),
SetBitrate(Bitrate),
AddEvent(EventData),
RemoveGlobalEvents,
SetConfig(Config),
Mute(bool),
Reconnect,
FullReconnect,
RebuildInterconnect,
Poison,
}
pub struct TrackContext {
pub track: Track,
pub handle: TrackHandle,
pub receiver: Receiver<TrackCommand>,
}

View File

@@ -0,0 +1,9 @@
#![allow(missing_docs)]
use crate::{driver::tasks::mixer::InternalTrack, tracks::TrackHandle};
#[allow(dead_code)] // We don't read because all we are doing is dropping.
pub enum DisposalMessage {
Track(Box<InternalTrack>),
Handle(TrackHandle),
}

View File

@@ -0,0 +1,34 @@
#![allow(missing_docs)]
use crate::{
events::{CoreContext, EventData, EventStore},
tracks::{LoopState, PlayMode, ReadyState, TrackHandle, TrackState},
};
use std::time::Duration;
pub enum EventMessage {
// Event related.
// Track events should fire off the back of state changes.
AddGlobalEvent(EventData),
AddTrackEvent(usize, EventData),
FireCoreEvent(CoreContext),
RemoveGlobalEvents,
AddTrack(EventStore, TrackState, TrackHandle),
ChangeState(usize, TrackStateChange),
RemoveAllTracks,
Tick,
Poison,
}
#[derive(Debug)]
pub enum TrackStateChange {
Mode(PlayMode),
Volume(f32),
Position(Duration),
// Bool indicates user-set.
Loops(LoopState, bool),
Total(TrackState),
Ready(ReadyState),
}

View File

@@ -0,0 +1,65 @@
#![allow(missing_docs)]
#[cfg(feature = "receive")]
use super::UdpRxMessage;
use super::{Interconnect, TrackContext, WsMessage};
use crate::{
driver::{crypto::Cipher, Bitrate, Config, CryptoState},
input::{AudioStreamError, Compose, Parsed},
};
use flume::Sender;
use std::{
net::UdpSocket,
sync::{atomic::AtomicU16, Arc, RwLock},
};
use symphonia_core::{errors::Error as SymphoniaError, formats::SeekedTo};
pub struct MixerConnection {
pub cipher: Cipher,
pub crypto_state: CryptoState,
pub dave_session: Arc<RwLock<Option<davey::DaveSession>>>,
pub dave_protocol_version: Arc<AtomicU16>,
#[cfg(feature = "receive")]
pub udp_rx: Sender<UdpRxMessage>,
pub udp_tx: UdpSocket,
}
pub enum MixerMessage {
AddTrack(Box<TrackContext>),
SetTrack(Option<Box<TrackContext>>),
SetBitrate(Bitrate),
SetConfig(Config),
SetMute(bool),
SetConn(MixerConnection, u32),
Ws(Option<Sender<WsMessage>>),
DropConn,
ReplaceInterconnect(Interconnect),
RebuildEncoder,
Poison,
}
impl MixerMessage {
#[must_use]
pub fn is_mixer_maybe_live(&self) -> bool {
matches!(
self,
Self::AddTrack(_) | Self::SetTrack(Some(_)) | Self::SetConn(..)
)
}
}
pub enum MixerInputResultMessage {
CreateErr(Arc<AudioStreamError>),
ParseErr(Arc<SymphoniaError>),
Seek(
Parsed,
Option<Box<dyn Compose>>,
Result<SeekedTo, Arc<SymphoniaError>>,
),
Built(Parsed, Option<Box<dyn Compose>>),
}

View File

@@ -0,0 +1,55 @@
#![allow(missing_docs)]
mod core;
mod disposal;
mod events;
mod mixer;
#[cfg(feature = "receive")]
mod udp_rx;
mod ws;
#[cfg(feature = "receive")]
pub use self::udp_rx::*;
pub use self::{core::*, disposal::*, events::*, mixer::*, ws::*};
use flume::Sender;
use tokio::spawn;
use tracing::trace;
#[derive(Clone, Debug)]
pub struct Interconnect {
pub core: Sender<CoreMessage>,
pub events: Sender<EventMessage>,
pub mixer: Sender<MixerMessage>,
}
impl Interconnect {
pub fn poison(&self) {
drop(self.events.send(EventMessage::Poison));
}
pub fn poison_all(&self) {
drop(self.mixer.send(MixerMessage::Poison));
self.poison();
}
pub fn restart_volatile_internals(&mut self) {
self.poison();
let (evt_tx, evt_rx) = flume::unbounded();
self.events = evt_tx;
spawn(async move {
trace!("Event processor restarted.");
super::events::runner(evt_rx).await;
trace!("Event processor finished.");
});
// Make mixer aware of new targets...
drop(
self.mixer
.send(MixerMessage::ReplaceInterconnect(self.clone())),
);
}
}

View File

@@ -0,0 +1,18 @@
#![allow(missing_docs)]
use super::Interconnect;
use crate::driver::DecodeConfig;
use dashmap::{DashMap, DashSet};
use serenity_voice_model::id::UserId;
pub enum UdpRxMessage {
SetConfig(DecodeConfig),
ReplaceInterconnect(Interconnect),
}
#[derive(Debug, Default)]
pub struct SsrcTracker {
pub disconnected_users: DashSet<UserId>,
pub user_ssrc_map: DashMap<UserId, u32>,
pub ssrc_user_map: DashMap<u32, UserId>,
}

View File

@@ -0,0 +1,12 @@
#![allow(missing_docs)]
use super::Interconnect;
use crate::{model::Event as GatewayEvent, ws::WsStream};
pub enum WsMessage {
Ws(Box<WsStream>),
ReplaceInterconnect(Interconnect),
SetKeepalive(f64),
Speaking(bool),
Deliver(GatewayEvent),
}

View File

@@ -0,0 +1,446 @@
use super::*;
/// Mix a track's audio stream into either the shared mixing buffer, or directly into the output
/// packet ("passthrough") when possible.
///
/// Passthrough is highest performance, but the source MUST be opus, have 20ms frames, and be the only
/// live track. In this case we copy the opus-encoded data with no changes. Otherwise, we fall back to
/// below.
///
/// There are a few functional requirements here for non-passthrough mixing that make it tricky:
/// * Input frame lengths are not congruent with what we need to send (i.e., 26.12ms in MP3 vs
/// needed 20ms).
/// * Input audio arrives at a different sample rate from required (i.e., 44.1 vs needed 48 kHz).
/// * Input data may not be `f32`s.
/// * Input data may not match stereo/mono of desired output.
///
/// All of the above challenges often happen at once. The rough pipeline in processing is:
///
/// until source end or 20 ms taken:
/// (use previous frame 'til empty / get new frame) -> [resample] -> [audio += vol * (sample as f32)]
///
/// Typically, we mix between a subset of the input packet and the output buf because the 20ms window
/// straddles packet boundaries. If there's enough space AND 48kHz AND receive f32s, then we use a fast
/// path.
///
/// In the mono -> stereo case, we duplicate across all target channels. In stereo -> mono, we average
/// the samples from each channel.
///
/// To avoid needing to hold onto resampled data longer than one mix cycle, we take enough input samples
/// to fill a chunk of the mixer (e.g., 10ms == 20ms / 2) so that they will all be used.
///
/// This is a fairly annoying piece of code to reason about, mainly because you need to hold so many
/// internal positions into: the mix buffer, resample buffers, and previous/current packets
/// for a stream.
#[inline]
pub fn mix_symph_indiv(
// shared buffer to mix into.
symph_mix: &mut AudioBuffer<f32>,
// buffer to hold built up packet
resample_scratch: &mut AudioBuffer<f32>,
// the input stream to use
input: &mut Parsed,
// resampler state and positions into partially read packets
local_state: &mut DecodeState,
// volume of this source
volume: f32,
// window into the output UDP buffer to copy opus frames into.
// This is set to `Some` IF passthrough is possible (i.e., one live source).
mut opus_slot: Option<&mut [u8]>,
) -> (MixType, MixStatus) {
let mut samples_written = 0;
let mut resample_in_progress = false;
let mut track_status = MixStatus::Live;
let codec_type = input.decoder.codec_params().codec;
resample_scratch.clear();
while samples_written != MONO_FRAME_SIZE {
// fetch a packet: either in progress, passthrough (early exit), or
let source_packet = if local_state.inner_pos != 0 {
Some(input.decoder.last_decoded())
} else if let Ok(pkt) = input.format.next_packet() {
if pkt.track_id() != input.track_id {
continue;
}
let buf = pkt.buf();
// Opus packet passthrough special case.
if codec_type == CODEC_TYPE_OPUS && local_state.passthrough != Passthrough::Block {
if let Some(slot) = opus_slot.as_mut() {
let sample_ct = if buf.is_empty() || buf.len() > i32::MAX as usize {
None
} else {
Some(opus2::packet::get_nb_samples(buf, SAMPLE_RATE))
};
// We don't actually block passthrough until a few violations are
// seen. The main one is that most Opus tracks end on a sub-20ms
// frame, particularly on Youtube.
// However, a frame that's bigger than the target buffer is an instant block.
let buf_size_fatal = buf.len() >= slot.len();
if match sample_ct {
Some(Ok(MONO_FRAME_SIZE)) => true,
_ => !local_state.record_and_check_passthrough_strike_final(buf_size_fatal),
} {
slot.write_all(buf)
.expect("Bounds check performed, and failure will block passthrough.");
return (MixType::Passthrough(buf.len()), MixStatus::Live);
}
}
}
input
.decoder
.decode(&pkt)
.map_err(|e| {
track_status = e.into();
})
.ok()
} else {
track_status = MixStatus::Ended;
None
};
// Cleanup: failed to get the next packet, but still have to convert and mix scratch.
if source_packet.is_none() {
if resample_in_progress {
// fill up remainder of buf with zeroes, resample, mix
let (chan_c, resampler, rs_out_buf) = local_state.resampler.as_mut().unwrap();
let in_len = resample_scratch.frames();
let to_render = resampler.input_frames_next().saturating_sub(in_len);
if to_render != 0 {
resample_scratch.render_reserved(Some(to_render));
for plane in resample_scratch.planes_mut().planes() {
for val in &mut plane[in_len..] {
*val = 0.0f32;
}
}
}
// Luckily, we make use of the WHOLE input buffer here.
resampler
.process_into_buffer(
&resample_scratch.planes().planes()[..*chan_c],
rs_out_buf,
None,
)
.unwrap();
// Calculate true end position using sample rate math
let ratio = (rs_out_buf[0].len() as f32) / (resample_scratch.frames() as f32);
let out_samples = (ratio * (in_len as f32)).round() as usize;
mix_resampled(rs_out_buf, symph_mix, samples_written, volume);
samples_written += out_samples;
}
break;
}
let source_packet = source_packet.unwrap();
let in_rate = source_packet.spec().rate;
let pkt_frames = source_packet.frames();
if pkt_frames == 0 {
continue;
}
if in_rate == SAMPLE_RATE_RAW as u32 {
// No need to resample: mix as standard.
let samples_marched = mix_over_ref(
&source_packet,
symph_mix,
local_state.inner_pos,
samples_written,
volume,
);
samples_written += samples_marched;
local_state.inner_pos += samples_marched;
local_state.inner_pos %= pkt_frames;
} else {
// NOTE: this should NEVER change in one stream.
let chan_c = source_packet.spec().channels.count();
let (_, resampler, rs_out_buf) = local_state.resampler.get_or_insert_with(|| {
// TODO: integ. error handling here.
let resampler = FftFixedOut::new(
in_rate as usize,
SAMPLE_RATE_RAW,
RESAMPLE_OUTPUT_FRAME_SIZE,
4,
chan_c,
)
.expect("Failed to create resampler.");
let out_buf = resampler.output_buffer_allocate(true);
(chan_c, resampler, out_buf)
});
let inner_pos = local_state.inner_pos;
let needed_in_frames = resampler.input_frames_next();
let available_frames = pkt_frames - inner_pos;
let force_copy = resample_in_progress || needed_in_frames > available_frames;
if (!force_copy) && matches!(source_packet, AudioBufferRef::F32(_)) {
// This is the only case where we can pull off a straight resample...
// I would really like if this could be a slice of slices,
// but the technology just isn't there yet. And I don't feel like
// writing unsafe transformations to do so.
// NOTE: if let needed as if-let && {bool} is nightly only.
if let AudioBufferRef::F32(s_pkt) = source_packet {
let refs: Vec<&[f32]> = s_pkt
.planes()
.planes()
.iter()
.map(|s| &s[inner_pos..][..needed_in_frames])
.collect();
local_state.inner_pos += needed_in_frames;
local_state.inner_pos %= pkt_frames;
resampler
.process_into_buffer(&refs, rs_out_buf, None)
.unwrap();
} else {
unreachable!()
}
} else {
// We either lack enough samples, or have the wrong data format, forcing
// a conversion/copy into the buffer.
let old_scratch_len = resample_scratch.frames();
let missing_frames = needed_in_frames - old_scratch_len;
let frames_to_take = available_frames.min(missing_frames);
resample_scratch.render_reserved(Some(frames_to_take));
copy_into_resampler(
&source_packet,
resample_scratch,
inner_pos,
old_scratch_len,
frames_to_take,
);
local_state.inner_pos += frames_to_take;
local_state.inner_pos %= pkt_frames;
if resample_scratch.frames() == needed_in_frames {
resampler
.process_into_buffer(
&resample_scratch.planes().planes()[..chan_c],
rs_out_buf,
None,
)
.unwrap();
resample_scratch.clear();
resample_in_progress = false;
} else {
// Not enough data to fill the resampler: fetch more.
resample_in_progress = true;
continue;
}
}
let samples_marched = mix_resampled(rs_out_buf, symph_mix, samples_written, volume);
samples_written += samples_marched;
}
}
(MixType::MixedPcm(samples_written), track_status)
}
#[inline]
fn mix_over_ref(
source: &AudioBufferRef<'_>,
target: &mut AudioBuffer<f32>,
source_pos: usize,
dest_pos: usize,
volume: f32,
) -> usize {
match source {
AudioBufferRef::U8(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
AudioBufferRef::U16(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
AudioBufferRef::U24(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
AudioBufferRef::U32(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
AudioBufferRef::S8(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
AudioBufferRef::S16(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
AudioBufferRef::S24(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
AudioBufferRef::S32(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
AudioBufferRef::F32(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
AudioBufferRef::F64(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
}
}
#[inline]
fn mix_symph_buffer<S>(
source: &AudioBuffer<S>,
target: &mut AudioBuffer<f32>,
source_pos: usize,
dest_pos: usize,
volume: f32,
) -> usize
where
S: Sample + IntoSample<f32>,
{
// mix in source_packet[inner_pos..] til end of EITHER buffer.
let src_usable = source.frames() - source_pos;
let tgt_usable = target.frames() - dest_pos;
let mix_ct = src_usable.min(tgt_usable);
let target_chans = target.spec().channels.count();
let target_mono = target_chans == 1;
let source_chans = source.spec().channels.count();
let source_mono = source_chans == 1;
let source_planes = source.planes();
let source_raw_planes = source_planes.planes();
if source_mono {
// mix this signal into *all* output channels at req'd volume.
let source_plane = source_raw_planes[0];
for d_plane in &mut (*target.planes_mut().planes()) {
for (d, s) in d_plane[dest_pos..dest_pos + mix_ct]
.iter_mut()
.zip(source_plane[source_pos..source_pos + mix_ct].iter())
{
*d += volume * (*s).into_sample();
}
}
} else if target_mono {
// mix all signals into the one target channel: reduce aggregate volume
// by n_channels.
let vol_adj = 1.0 / (source_chans as f32);
let mut t_planes = target.planes_mut();
let d_plane = &mut *t_planes.planes()[0];
for s_plane in source_raw_planes {
for (d, s) in d_plane[dest_pos..dest_pos + mix_ct]
.iter_mut()
.zip(s_plane[source_pos..source_pos + mix_ct].iter())
{
*d += volume * vol_adj * (*s).into_sample();
}
}
} else {
// stereo -> stereo: don't change volume, map input -> output channels w/ no duplication
for (d_plane, s_plane) in (*target.planes_mut().planes())
.iter_mut()
.zip(source_raw_planes[..].iter())
{
for (d, s) in d_plane[dest_pos..dest_pos + mix_ct]
.iter_mut()
.zip(s_plane[source_pos..source_pos + mix_ct].iter())
{
*d += volume * (*s).into_sample();
}
}
}
mix_ct
}
#[inline]
fn mix_resampled(
source: &[Vec<f32>],
target: &mut AudioBuffer<f32>,
dest_pos: usize,
volume: f32,
) -> usize {
let mix_ct = source[0].len();
let target_chans = target.spec().channels.count();
let target_mono = target_chans == 1;
let source_chans = source.len();
let source_mono = source_chans == 1;
// see `mix_symph_buffer` for explanations of stereo<->mono logic.
if source_mono {
let source_plane = &source[0];
for d_plane in &mut (*target.planes_mut().planes()) {
for (d, s) in d_plane[dest_pos..dest_pos + mix_ct]
.iter_mut()
.zip(source_plane)
{
*d += volume * s;
}
}
} else if target_mono {
let vol_adj = 1.0 / (source_chans as f32);
let mut t_planes = target.planes_mut();
let d_plane = &mut *t_planes.planes()[0];
for s_plane in source {
for (d, s) in d_plane[dest_pos..dest_pos + mix_ct].iter_mut().zip(s_plane) {
*d += volume * vol_adj * s;
}
}
} else {
for (d_plane, s_plane) in (*target.planes_mut().planes())
.iter_mut()
.zip(source[..].iter())
{
for (d, s) in d_plane[dest_pos..dest_pos + mix_ct].iter_mut().zip(s_plane) {
*d += volume * (*s);
}
}
}
mix_ct
}
#[inline]
pub(crate) fn copy_into_resampler(
source: &AudioBufferRef<'_>,
target: &mut AudioBuffer<f32>,
source_pos: usize,
dest_pos: usize,
len: usize,
) -> usize {
match source {
AudioBufferRef::U8(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
AudioBufferRef::U16(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
AudioBufferRef::U24(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
AudioBufferRef::U32(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
AudioBufferRef::S8(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
AudioBufferRef::S16(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
AudioBufferRef::S24(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
AudioBufferRef::S32(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
AudioBufferRef::F32(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
AudioBufferRef::F64(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
}
}
#[inline]
fn copy_symph_buffer<S>(
source: &AudioBuffer<S>,
target: &mut AudioBuffer<f32>,
source_pos: usize,
dest_pos: usize,
len: usize,
) -> usize
where
S: Sample + IntoSample<f32>,
{
for (d_plane, s_plane) in (*target.planes_mut().planes())
.iter_mut()
.zip(source.planes().planes()[..].iter())
{
for (d, s) in d_plane[dest_pos..dest_pos + len]
.iter_mut()
.zip(s_plane[source_pos..source_pos + len].iter())
{
*d = (*s).into_sample();
}
}
len
}

View File

@@ -0,0 +1,884 @@
pub mod mix_logic;
mod pool;
mod result;
pub mod state;
pub mod track;
mod util;
use pool::*;
use result::*;
use state::*;
pub use track::*;
use super::{
disposal::DisposalThread,
error::{Error, Result},
message::*,
};
use crate::{
constants::*,
driver::{CryptoMode, MixMode},
events::EventStore,
input::{Input, Parsed},
tracks::{Action, LoopState, PlayError, PlayMode, TrackCommand, TrackHandle, TrackState, View},
Config,
};
use discortp::{
discord::MutableKeepalivePacket,
rtp::{MutableRtpPacket, RtpPacket},
MutablePacket,
};
use flume::{Receiver, SendError, Sender, TryRecvError};
use opus2::{Application as CodingMode, Bitrate, Encoder as OpusEncoder, SoftClip};
use rand::random;
use rubato::{FftFixedOut, Resampler};
use std::{
io::Write,
result::Result as StdResult,
sync::{atomic::Ordering, Arc},
time::{Duration, Instant},
};
use symphonia_core::{
audio::{AudioBuffer, AudioBufferRef, Layout, SampleBuffer, Signal, SignalSpec},
codecs::CODEC_TYPE_OPUS,
conv::IntoSample,
formats::SeekTo,
sample::Sample,
units::Time,
};
use tokio::runtime::Handle;
use tracing::error;
#[cfg(test)]
use crate::driver::test_config::{OutputMessage, OutputMode};
#[cfg(test)]
use discortp::Packet as _;
pub struct Mixer {
pub bitrate: Bitrate,
pub config: Arc<Config>,
pub conn_active: Option<MixerConnection>,
pub deadline: Instant,
pub disposer: DisposalThread,
pub encoder: OpusEncoder,
pub interconnect: Interconnect,
pub mix_rx: Receiver<MixerMessage>,
pub muted: bool,
// pub packet: [u8; VOICE_PACKET_MAX],
pub prevent_events: bool,
pub silence_frames: u8,
pub soft_clip: SoftClip,
thread_pool: BlockyTaskPool,
pub ws: Option<Sender<WsMessage>>,
pub keepalive_deadline: Instant,
pub keepalive_packet: [u8; MutableKeepalivePacket::minimum_packet_size()],
pub tracks: Vec<InternalTrack>,
track_handles: Vec<TrackHandle>,
sample_buffer: SampleBuffer<f32>,
symph_mix: AudioBuffer<f32>,
resample_scratch: AudioBuffer<f32>,
#[cfg(test)]
pub remaining_loops: Option<u64>,
#[cfg(test)]
raw_msg: Option<OutputMessage>,
}
fn new_encoder(bitrate: Bitrate, mix_mode: MixMode) -> Result<OpusEncoder> {
let mut encoder = OpusEncoder::new(SAMPLE_RATE, mix_mode.to_opus(), CodingMode::Audio)?;
encoder.set_bitrate(bitrate)?;
Ok(encoder)
}
impl Mixer {
#[must_use]
pub fn new(
mix_rx: Receiver<MixerMessage>,
async_handle: Handle,
interconnect: Interconnect,
config: Config,
) -> Self {
let bitrate = DEFAULT_BITRATE;
let encoder = new_encoder(bitrate, config.mix_mode)
.expect("Failed to create encoder in mixing thread with known-good values.");
let soft_clip = SoftClip::new(config.mix_mode.to_opus());
let keepalive_packet = [0u8; MutableKeepalivePacket::minimum_packet_size()];
let tracks = Vec::with_capacity(usize::from(1.max(config.preallocated_tracks)));
let track_handles = Vec::with_capacity(usize::from(1.max(config.preallocated_tracks)));
let thread_pool = BlockyTaskPool::new(async_handle);
let symph_layout = config.mix_mode.symph_layout();
let disposer = config.disposer.clone().unwrap_or_default();
let config = config.into();
let sample_buffer = SampleBuffer::<f32>::new(
MONO_FRAME_SIZE as u64,
symphonia_core::audio::SignalSpec::new_with_layout(
SAMPLE_RATE_RAW as u32,
symph_layout,
),
);
let symph_mix = AudioBuffer::<f32>::new(
MONO_FRAME_SIZE as u64,
symphonia_core::audio::SignalSpec::new_with_layout(
SAMPLE_RATE_RAW as u32,
symph_layout,
),
);
let resample_scratch = AudioBuffer::<f32>::new(
MONO_FRAME_SIZE as u64,
SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, Layout::Stereo),
);
let deadline = Instant::now();
Self {
bitrate,
config,
conn_active: None,
deadline,
disposer,
encoder,
interconnect,
mix_rx,
muted: false,
prevent_events: false,
silence_frames: 0,
soft_clip,
thread_pool,
ws: None,
keepalive_deadline: deadline,
keepalive_packet,
tracks,
track_handles,
sample_buffer,
symph_mix,
resample_scratch,
#[cfg(test)]
remaining_loops: None,
#[cfg(test)]
raw_msg: None,
}
}
fn set_bitrate(&mut self, bitrate: Bitrate) -> Result<()> {
self.encoder.set_bitrate(bitrate).map_err(Into::into)
}
pub(crate) fn do_rebuilds(
&mut self,
event_failure: bool,
conn_failure: bool,
) -> StdResult<(), SendError<CoreMessage>> {
// event failure? rebuild interconnect.
// ws or udp failure? full connect
// (soft reconnect is covered by the ws task.)
//
// in both cases, send failure is fatal,
// but will only occur on disconnect.
if event_failure {
self.rebuild_interconnect()?;
}
if conn_failure {
self.full_reconnect_gateway()?;
}
Ok(())
}
pub(crate) fn rebuild_interconnect(&mut self) -> StdResult<(), SendError<CoreMessage>> {
self.prevent_events = true;
self.interconnect
.core
.send(CoreMessage::RebuildInterconnect)
}
pub(crate) fn full_reconnect_gateway(&mut self) -> StdResult<(), SendError<CoreMessage>> {
self.conn_active = None;
self.interconnect.core.send(CoreMessage::FullReconnect)
}
#[inline]
pub(crate) fn handle_message(
&mut self,
msg: MixerMessage,
packet: &mut [u8],
) -> (bool, bool, bool) {
let mut events_failure = false;
let mut conn_failure = false;
let mut should_exit = false;
let error = match msg {
MixerMessage::AddTrack(t) => self.add_track(*t),
MixerMessage::SetTrack(t) => {
self.tracks.clear();
self.track_handles.clear();
let mut out = self.fire_event(EventMessage::RemoveAllTracks);
if let Some(t) = t {
// Do this unconditionally: this affects local state infallibly,
// with the event installation being the remote part.
if let Err(e) = self.add_track(*t) {
out = Err(e);
}
}
out
},
MixerMessage::SetBitrate(b) => {
self.bitrate = b;
if let Err(e) = self.set_bitrate(b) {
error!("Failed to update bitrate {:?}", e);
}
Ok(())
},
MixerMessage::SetMute(m) => {
self.muted = m;
Ok(())
},
MixerMessage::SetConn(conn, ssrc) => {
self.conn_active = Some(conn);
let mut rtp = MutableRtpPacket::new(packet).expect(
"Too few bytes in self.packet for RTP header.\
(Blame: VOICE_PACKET_MAX?)",
);
rtp.set_ssrc(ssrc);
rtp.set_sequence(random::<u16>().into());
rtp.set_timestamp(random::<u32>().into());
self.deadline = Instant::now();
self.update_keepalive(ssrc);
Ok(())
},
MixerMessage::DropConn => {
self.conn_active = None;
Ok(())
},
MixerMessage::ReplaceInterconnect(i) => {
self.prevent_events = false;
if let Some(ws) = &self.ws {
conn_failure |= ws.send(WsMessage::ReplaceInterconnect(i.clone())).is_err();
}
#[cfg(feature = "receive")]
if let Some(conn) = &self.conn_active {
conn_failure |= conn
.udp_rx
.send(UdpRxMessage::ReplaceInterconnect(i.clone()))
.is_err();
}
self.interconnect = i;
self.rebuild_tracks()
},
MixerMessage::SetConfig(new_config) => {
if new_config.mix_mode != self.config.mix_mode {
self.soft_clip = SoftClip::new(new_config.mix_mode.to_opus());
if let Ok(enc) = new_encoder(self.bitrate, new_config.mix_mode) {
self.encoder = enc;
} else {
self.bitrate = DEFAULT_BITRATE;
self.encoder = new_encoder(self.bitrate, new_config.mix_mode)
.expect("Failed fallback rebuild of OpusEncoder with safe inputs.");
}
let sl = new_config.mix_mode.symph_layout();
self.sample_buffer = SampleBuffer::<f32>::new(
MONO_FRAME_SIZE as u64,
SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, sl),
);
self.symph_mix = AudioBuffer::<f32>::new(
MONO_FRAME_SIZE as u64,
SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, sl),
);
}
self.config = Arc::new(
#[cfg(feature = "receive")]
new_config.clone(),
#[cfg(not(feature = "receive"))]
new_config,
);
let preallocated_tracks = usize::from(self.config.preallocated_tracks);
if self.tracks.capacity() < preallocated_tracks {
self.tracks.reserve(preallocated_tracks - self.tracks.len());
}
#[cfg(feature = "receive")]
if let Some(conn) = &self.conn_active {
if let crate::driver::DecodeMode::Decode(decode_config) = new_config.decode_mode
{
let msg = UdpRxMessage::SetConfig(decode_config);
conn_failure |= conn.udp_rx.send(msg).is_err();
}
}
Ok(())
},
MixerMessage::RebuildEncoder => match new_encoder(self.bitrate, self.config.mix_mode) {
Ok(encoder) => {
self.encoder = encoder;
Ok(())
},
Err(e) => {
error!("Failed to rebuild encoder. Resetting bitrate. {:?}", e);
self.bitrate = DEFAULT_BITRATE;
self.encoder = new_encoder(self.bitrate, self.config.mix_mode)
.expect("Failed fallback rebuild of OpusEncoder with safe inputs.");
Ok(())
},
},
MixerMessage::Ws(new_ws_handle) => {
self.ws = new_ws_handle;
if let Err(e) = self.send_gateway_speaking() {
conn_failure |= e.should_trigger_connect();
}
Ok(())
},
MixerMessage::Poison => {
should_exit = true;
Ok(())
},
};
if let Err(e) = error {
events_failure |= e.should_trigger_interconnect_rebuild();
conn_failure |= e.should_trigger_connect();
}
(events_failure, conn_failure, should_exit)
}
pub(crate) fn update_keepalive(&mut self, ssrc: u32) {
let mut ka = MutableKeepalivePacket::new(&mut self.keepalive_packet[..])
.expect("FATAL: Insufficient bytes given to keepalive packet.");
ka.set_ssrc(ssrc);
self.keepalive_deadline = self.deadline + UDP_KEEPALIVE_GAP;
}
#[inline]
pub(crate) fn fire_event(&self, event: EventMessage) -> Result<()> {
// As this task is responsible for noticing the potential death of an event context,
// it's responsible for not forcibly recreating said context repeatedly.
if !self.prevent_events {
self.interconnect.events.send(event)?;
}
Ok(())
}
#[inline]
pub fn add_track(&mut self, track: TrackContext) -> Result<()> {
let (track, evts, state, handle) = InternalTrack::decompose_track(track);
self.tracks.push(track);
self.track_handles.push(handle.clone());
self.interconnect
.events
.send(EventMessage::AddTrack(evts, state, handle))?;
Ok(())
}
// rebuilds the event thread's view of each track, in event of a full rebuild.
#[inline]
fn rebuild_tracks(&mut self) -> Result<()> {
for (track, handle) in self.tracks.iter().zip(self.track_handles.iter()) {
let evts = EventStore::default();
let state = track.state();
let handle = handle.clone();
self.interconnect
.events
.send(EventMessage::AddTrack(evts, state, handle))?;
}
Ok(())
}
#[inline]
pub(crate) fn audio_commands_events(&mut self) -> Result<()> {
// Apply user commands.
for (i, track) in self.tracks.iter_mut().enumerate() {
// This causes fallible event system changes,
// but if the event thread has died then we'll certainly
// detect that on the tick later.
// Changes to play state etc. MUST all be handled.
let action = track.process_commands(i, &self.interconnect);
if let Some(req) = action.seek_point {
track.seek(
i,
req,
&self.interconnect,
&self.thread_pool,
&self.config,
self.prevent_events,
);
}
if let Some(callback) = action.make_playable {
if let Err(e) = track.get_or_ready_input(
i,
&self.interconnect,
&self.thread_pool,
&self.config,
self.prevent_events,
) {
track.callbacks.make_playable = Some(callback);
if let Some(fail) = e.as_user() {
track.playing = PlayMode::Errored(fail);
}
if let Some(req) = e.into_seek_request() {
track.seek(
i,
req,
&self.interconnect,
&self.thread_pool,
&self.config,
self.prevent_events,
);
}
} else {
// Track is already ready: don't register callback and just act.
drop(callback.send(Ok(())));
}
}
}
let mut i = 0;
while i < self.tracks.len() {
let track = self
.tracks
.get_mut(i)
.expect("Tried to remove an illegal track index.");
if track.playing.is_done() {
let p_state = track.playing.clone();
let to_drop = self.tracks.swap_remove(i);
self.disposer
.dispose(DisposalMessage::Track(Box::new(to_drop)));
let to_drop = self.track_handles.swap_remove(i);
self.disposer.dispose(DisposalMessage::Handle(to_drop));
self.fire_event(EventMessage::ChangeState(
i,
TrackStateChange::Mode(p_state),
))?;
} else {
i += 1;
}
}
// Tick -- receive side also handles removals in same manner after it increments
// times etc.
self.fire_event(EventMessage::Tick)?;
Ok(())
}
#[cfg(test)]
#[inline]
pub(crate) fn test_signal_empty_tick(&self) {
match &self.config.override_connection {
Some(OutputMode::Raw(tx)) =>
drop(tx.send(crate::driver::test_config::TickMessage::NoEl)),
Some(OutputMode::Rtp(tx)) =>
drop(tx.send(crate::driver::test_config::TickMessage::NoEl)),
None => {},
}
}
pub fn crypto_mode(&self) -> CryptoMode {
let mode = self.conn_active.as_ref().map(|v| v.crypto_state.kind());
if cfg!(not(test)) {
mode.expect("Shouldn't be mixing packets without access to a cipher + UDP dest.")
} else {
mode.unwrap_or_else(|| self.config.crypto_mode)
}
}
#[inline]
pub fn mix_and_build_packet(&mut self, packet: &mut [u8]) -> Result<usize> {
// symph_mix is an `AudioBuffer` (planar format), we need to convert this
// later into an interleaved `SampleBuffer` for libopus.
self.symph_mix.clear();
self.symph_mix.render_reserved(Some(MONO_FRAME_SIZE));
self.resample_scratch.clear();
// Walk over all the audio files, combining into one audio frame according
// to volume, play state, etc.
let mut mix_len = {
let out = self.mix_tracks(packet);
self.sample_buffer.copy_interleaved_typed(&self.symph_mix);
out
};
if self.muted {
mix_len = MixType::MixedPcm(0);
}
// Explicit "Silence" frame handling: if there is no mixed data, we must send
// ~5 frames of silence (unless another good audio frame appears) before we
// stop sending RTP frames.
if mix_len == MixType::MixedPcm(0) {
if self.silence_frames > 0 {
self.silence_frames -= 1;
let mut rtp = MutableRtpPacket::new(packet).expect(
"FATAL: Too few bytes in self.packet for RTP header.\
(Blame: VOICE_PACKET_MAX?)",
);
let payload = rtp.payload_mut();
let pre_len = self.crypto_mode().payload_prefix_len();
payload[pre_len..pre_len + SILENT_FRAME.len()].copy_from_slice(&SILENT_FRAME[..]);
mix_len = MixType::Passthrough(SILENT_FRAME.len());
} else {
// Per official guidelines, send 5x silence BEFORE we stop speaking.
return Ok(0);
}
} else {
self.silence_frames = 5;
if let MixType::MixedPcm(n) = mix_len {
if self.config.use_softclip {
self.soft_clip.apply(
&mut self.sample_buffer.samples_mut()
[..n * self.config.mix_mode.channels()],
);
}
}
}
// For the benefit of test cases, send the raw un-RTP'd data.
#[cfg(test)]
let out = if let Some(OutputMode::Raw(_)) = &self.config.override_connection {
// This case has been handled before buffer clearing above.
let msg = match mix_len {
MixType::Passthrough(len) if len == SILENT_FRAME.len() => OutputMessage::Silent,
MixType::Passthrough(len) => {
let rtp = RtpPacket::new(packet).expect(
"FATAL: Too few bytes in self.packet for RTP header.\
(Blame: VOICE_PACKET_MAX?)",
);
let payload = rtp.payload();
let opus_frame =
(payload[self.crypto_mode().payload_prefix_len()..][..len]).to_vec();
OutputMessage::Passthrough(opus_frame)
},
MixType::MixedPcm(_) => OutputMessage::Mixed(
self.sample_buffer.samples()[..self.config.mix_mode.sample_count_in_frame()]
.to_vec(),
),
};
self.raw_msg = Some(msg);
Ok(1)
} else {
self.prep_packet(mix_len, packet)
};
#[cfg(not(test))]
let out = self.prep_packet(mix_len, packet);
// Zero out all planes of the mix buffer if any audio was written.
if matches!(mix_len, MixType::MixedPcm(a) if a > 0) {
for plane in self.symph_mix.planes_mut().planes() {
plane.fill(0.0);
}
}
out
}
#[inline]
fn prep_packet(&mut self, mix_len: MixType, packet: &mut [u8]) -> Result<usize> {
let send_buffer = self.sample_buffer.samples();
let conn = self
.conn_active
.as_mut()
.expect("Shouldn't be mixing packets without access to a cipher + UDP dest.");
let mut rtp = MutableRtpPacket::new(packet).expect(
"FATAL: Too few bytes in self.packet for RTP header.\
(Blame: VOICE_PACKET_MAX?)",
);
let payload = rtp.payload_mut();
let crypto_mode = conn.crypto_state.kind();
let first_payload_byte = crypto_mode.payload_prefix_len();
let total_payload_space = payload.len() - crypto_mode.payload_suffix_len();
// If passthrough, Opus payload in place already.
// Else encode into buffer with space for AEAD encryption headers.
let mut payload_len = match mix_len {
MixType::Passthrough(opus_len) => opus_len,
MixType::MixedPcm(_samples) => self.encoder.encode_float(
&send_buffer[..self.config.mix_mode.sample_count_in_frame()],
&mut payload[first_payload_byte..total_payload_space],
)?,
};
if conn.dave_protocol_version.load(Ordering::Relaxed) != 0 {
if let Some(ref mut dave_session) = *conn.dave_session.write().unwrap() {
if dave_session.is_ready() {
let encrypted = dave_session
.encrypt_opus(
&payload[first_payload_byte..first_payload_byte + payload_len],
)?
.into_owned();
payload_len = encrypted.len();
payload[first_payload_byte..first_payload_byte + payload_len]
.copy_from_slice(&encrypted);
}
}
}
let final_payload_size = conn
.crypto_state
.write_packet_nonce(&mut rtp, first_payload_byte + payload_len);
// Packet encryption ignored in test modes.
#[cfg(not(test))]
let encrypt = true;
#[cfg(test)]
let encrypt = self.config.override_connection.is_none();
if encrypt {
conn.cipher
.encrypt_pkt_in_place(&mut rtp, final_payload_size)?;
}
Ok(RtpPacket::minimum_packet_size() + final_payload_size)
}
#[inline]
pub(crate) fn send_packet(&self, packet: &[u8]) -> Result<()> {
#[cfg(test)]
let send_status = if let Some(OutputMode::Raw(tx)) = &self.config.override_connection {
// This case has been handled before buffer clearing in `mix_and_build_packet`.
drop(tx.send(self.raw_msg.clone().unwrap().into()));
Ok(())
} else {
self.send_packet_(packet)
};
#[cfg(not(test))]
let send_status = self.send_packet_(packet);
send_status.or_else(Error::disarm_would_block)?;
Ok(())
}
#[inline]
fn send_packet_(&self, packet: &[u8]) -> Result<()> {
let conn = self
.conn_active
.as_ref()
.expect("Shouldn't be mixing packets without access to a cipher + UDP dest.");
#[cfg(test)]
if let Some(OutputMode::Rtp(tx)) = &self.config.override_connection {
// Test mode: send unencrypted (compressed) packets to local receiver.
drop(tx.send(packet.to_vec().into()));
} else {
conn.udp_tx.send(packet)?;
}
#[cfg(not(test))]
{
// Normal operation: send encrypted payload to UDP Tx task.
conn.udp_tx.send(packet)?;
}
Ok(())
}
#[inline]
pub(crate) fn check_and_send_keepalive(&mut self, now: Option<Instant>) -> Result<()> {
if let Some(conn) = self.conn_active.as_mut() {
let now = now.unwrap_or_else(Instant::now);
if now >= self.keepalive_deadline {
conn.udp_tx.send(&self.keepalive_packet)?;
self.keepalive_deadline += UDP_KEEPALIVE_GAP;
}
}
Ok(())
}
#[inline]
pub(crate) fn send_gateway_speaking(&self) -> Result<()> {
if let Some(ws) = &self.ws {
ws.send(WsMessage::Speaking(true))?;
}
Ok(())
}
#[inline]
pub(crate) fn send_gateway_not_speaking(&self) {
if let Some(ws) = &self.ws {
// NOTE: this explicit `drop` should prevent a catastrophic thread pileup.
// A full reconnect might cause an inner closed connection.
// It's safer to leave the central task to clean this up and
// pass the mixer a new channel.
drop(ws.send(WsMessage::Speaking(false)));
}
}
#[inline]
fn mix_tracks(&mut self, packet: &mut [u8]) -> MixType {
// Get a slice of bytes to write in data for Opus packet passthrough.
let mut rtp = MutableRtpPacket::new(packet).expect(
"FATAL: Too few bytes in self.packet for RTP header.\
(Blame: VOICE_PACKET_MAX?)",
);
let payload = rtp.payload_mut();
let opus_frame = &mut payload[self.crypto_mode().payload_prefix_len()..];
// Opus frame passthrough.
// This requires that we have only one PLAYING track, who has volume 1.0, and an
// Opus codec type (verified later in mix_symph_indiv).
//
// We *could* cache the number of live tracks separately, but that makes this
// quite fragile given all the ways a user can alter the PlayMode.
let mut num_live = 0;
let mut last_live_vol = 1.0;
for track in &self.tracks {
if track.playing.is_playing() {
num_live += 1;
last_live_vol = track.volume;
}
}
let do_passthrough = num_live == 1 && (last_live_vol - 1.0).abs() < f32::EPSILON;
let mut len = 0;
for (i, track) in self.tracks.iter_mut().enumerate() {
let vol = track.volume;
// This specifically tries to get tracks who are "preparing",
// so that event handlers and the like can all be fired without
// the track being in a `Play` state.
if !track.should_check_input() {
continue;
}
let should_play = track.playing.is_playing();
let input = track.get_or_ready_input(
i,
&self.interconnect,
&self.thread_pool,
&self.config,
self.prevent_events,
);
let (input, mix_state) = match input {
Ok(i) => i,
Err(InputReadyingError::Waiting) => continue,
Err(InputReadyingError::NeedsSeek(req)) => {
track.seek(
i,
req,
&self.interconnect,
&self.thread_pool,
&self.config,
self.prevent_events,
);
continue;
},
// TODO: allow for retry in given time.
Err(e) => {
if let Some(fail) = e.as_user() {
track.playing = PlayMode::Errored(fail);
}
continue;
},
};
// Now that we have dealt with potential errors in preparing tracks,
// only do any mixing if the track is to be played!
if !should_play {
continue;
}
let (mix_type, status) = mix_logic::mix_symph_indiv(
&mut self.symph_mix,
&mut self.resample_scratch,
input,
mix_state,
vol,
do_passthrough.then_some(&mut *opus_frame),
);
let return_here = if let MixType::MixedPcm(pcm_len) = mix_type {
len = len.max(pcm_len);
false
} else {
if mix_state.passthrough == Passthrough::Inactive {
input.decoder.reset();
}
mix_state.passthrough = Passthrough::Active;
true
};
// FIXME: allow Ended to trigger a seek/loop/revisit in the same mix cycle?
// Would this be possible with special-casing to mark some inputs as fast
// to recreate? Probably not doable in the general case.
match status {
MixStatus::Live => track.step_frame(),
MixStatus::Errored(e) =>
track.playing = PlayMode::Errored(PlayError::Decode(e.into())),
MixStatus::Ended if track.do_loop() => {
drop(self.track_handles[i].seek(Duration::default()));
if !self.prevent_events {
// position update is sent out later, when the seek concludes.
drop(self.interconnect.events.send(EventMessage::ChangeState(
i,
TrackStateChange::Loops(track.loops, false),
)));
}
},
MixStatus::Ended => {
track.end();
},
}
// This needs to happen here due to borrow checker shenanigans.
if return_here {
return mix_type;
}
}
MixType::MixedPcm(len)
}
}

View File

@@ -0,0 +1,141 @@
use super::util::copy_seek_to;
use crate::{
driver::tasks::message::MixerInputResultMessage,
input::{AudioStream, AudioStreamError, Compose, Input, LiveInput, Parsed},
Config,
};
use flume::Sender;
use rusty_pool::ThreadPool;
use std::{result::Result as StdResult, sync::Arc, time::Duration};
use symphonia_core::{
formats::{SeekMode, SeekTo},
io::MediaSource,
};
use tokio::runtime::Handle;
#[derive(Clone)]
pub struct BlockyTaskPool {
pool: ThreadPool,
handle: Handle,
}
impl BlockyTaskPool {
pub fn new(handle: Handle) -> Self {
Self {
pool: ThreadPool::new(0, 64, Duration::from_secs(5)),
handle,
}
}
pub fn create(
&self,
callback: Sender<MixerInputResultMessage>,
input: Input,
seek_time: Option<SeekTo>,
config: Arc<Config>,
) {
// Moves an Input from Lazy -> Live.
// We either do this on this pool, or move it to the tokio executor as the source requires.
// This takes a seek_time to pass on and execute *after* parsing (i.e., back-seek on
// read-only stream).
match input {
Input::Lazy(mut lazy) => {
let far_pool = self.clone();
if lazy.should_create_async() {
self.handle.spawn(async move {
let out = lazy.create_async().await;
far_pool.send_to_parse(out, lazy, callback, seek_time, config);
});
} else {
self.pool.execute(move || {
let out = lazy.create();
far_pool.send_to_parse(out, lazy, callback, seek_time, config);
});
}
},
Input::Live(live, maybe_create) =>
self.parse(config, callback, live, maybe_create, seek_time),
}
}
pub fn send_to_parse(
&self,
create_res: StdResult<AudioStream<Box<dyn MediaSource>>, AudioStreamError>,
rec: Box<dyn Compose>,
callback: Sender<MixerInputResultMessage>,
seek_time: Option<SeekTo>,
config: Arc<Config>,
) {
match create_res {
Ok(o) => {
self.parse(config, callback, LiveInput::Raw(o), Some(rec), seek_time);
},
Err(e) => {
drop(callback.send(MixerInputResultMessage::CreateErr(e.into())));
},
}
}
pub fn parse(
&self,
config: Arc<Config>,
callback: Sender<MixerInputResultMessage>,
input: LiveInput,
rec: Option<Box<dyn Compose>>,
seek_time: Option<SeekTo>,
) {
let pool_clone = self.clone();
self.pool.execute(move || {
match input.promote(config.codec_registry, config.format_registry) {
Ok(LiveInput::Parsed(parsed)) => match seek_time {
// If seek time is zero, then wipe it out.
// Some formats (MKV) make SeekTo(0) require a backseek to realign with the
// current page.
Some(seek_time) if !super::util::seek_to_is_zero(&seek_time) => {
pool_clone.seek(callback, parsed, rec, seek_time, false, config);
},
_ => {
drop(callback.send(MixerInputResultMessage::Built(parsed, rec)));
},
},
Ok(_) => unreachable!(),
Err(e) => {
drop(callback.send(MixerInputResultMessage::ParseErr(e.into())));
},
}
});
}
pub fn seek(
&self,
callback: Sender<MixerInputResultMessage>,
mut input: Parsed,
rec: Option<Box<dyn Compose>>,
seek_time: SeekTo,
// Not all of symphonia's formats bother to return SeekErrorKind::ForwardOnly.
// So, we need *this* flag.
backseek_needed: bool,
config: Arc<Config>,
) {
let pool_clone = self.clone();
self.pool.execute(move || match rec {
Some(rec) if (!input.supports_backseek) && backseek_needed => {
pool_clone.create(callback, Input::Lazy(rec), Some(seek_time), config);
},
_ => {
let seek_result = input
.format
.seek(SeekMode::Accurate, copy_seek_to(&seek_time));
input.decoder.reset();
drop(callback.send(MixerInputResultMessage::Seek(
input,
rec,
seek_result.map_err(Arc::new),
)));
},
});
}
}

View File

@@ -0,0 +1,55 @@
use crate::{
input::AudioStreamError,
tracks::{PlayError, SeekRequest},
};
use std::sync::Arc;
use symphonia_core::errors::Error as SymphoniaError;
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub enum MixType {
Passthrough(usize),
MixedPcm(usize),
}
pub enum MixStatus {
Live,
Ended,
Errored(SymphoniaError),
}
impl From<SymphoniaError> for MixStatus {
fn from(e: SymphoniaError) -> Self {
Self::Errored(e)
}
}
// The Symph errors are Arc'd here since if they come up, they will always
// be Arc'd anyway via into_user.
#[derive(Clone, Debug)]
pub enum InputReadyingError {
Parsing(Arc<SymphoniaError>),
Creation(Arc<AudioStreamError>),
Seeking(Arc<SymphoniaError>),
Dropped,
Waiting,
NeedsSeek(SeekRequest),
}
impl InputReadyingError {
pub fn as_user(&self) -> Option<PlayError> {
match self {
Self::Parsing(e) => Some(PlayError::Parse(e.clone())),
Self::Creation(e) => Some(PlayError::Create(e.clone())),
Self::Seeking(e) => Some(PlayError::Seek(e.clone())),
_ => None,
}
}
pub fn into_seek_request(self) -> Option<SeekRequest> {
if let Self::NeedsSeek(a) = self {
Some(a)
} else {
None
}
}
}

View File

@@ -0,0 +1,105 @@
use crate::{
constants::OPUS_PASSTHROUGH_STRIKE_LIMIT,
driver::tasks::message::*,
input::{Compose, Input, LiveInput, Metadata, Parsed},
tracks::{ReadyState, SeekRequest},
};
use flume::Receiver;
use rubato::FftFixedOut;
use std::time::Instant;
pub enum InputState {
NotReady(Input),
Preparing(PreparingInfo),
Ready(Parsed, Option<Box<dyn Compose>>),
}
impl InputState {
pub fn metadata(&mut self) -> Option<Metadata<'_>> {
if let Self::Ready(parsed, _) = self {
Some(parsed.into())
} else {
None
}
}
#[must_use]
pub fn ready_state(&self) -> ReadyState {
match self {
Self::NotReady(_) => ReadyState::Uninitialised,
Self::Preparing(_) => ReadyState::Preparing,
Self::Ready(_, _) => ReadyState::Playable,
}
}
}
impl From<Input> for InputState {
fn from(val: Input) -> Self {
match val {
a @ Input::Lazy(_) => Self::NotReady(a),
Input::Live(live, rec) => match live {
LiveInput::Parsed(p) => Self::Ready(p, rec),
other => Self::NotReady(Input::Live(other, rec)),
},
}
}
}
pub struct PreparingInfo {
#[allow(dead_code)]
/// Time this request was fired.
pub time: Instant,
/// Used to handle seek requests fired while a track was being created (or a seek was in progress).
pub queued_seek: Option<SeekRequest>,
/// Callback from the thread pool to indicate the result of creating/parsing this track.
pub callback: Receiver<MixerInputResultMessage>,
}
pub struct DecodeState {
pub inner_pos: usize,
pub resampler: Option<(usize, FftFixedOut<f32>, Vec<Vec<f32>>)>,
pub passthrough: Passthrough,
pub passthrough_violations: u8,
}
impl DecodeState {
pub fn reset(&mut self) {
self.inner_pos = 0;
self.resampler = None;
}
pub fn record_and_check_passthrough_strike_final(&mut self, fatal: bool) -> bool {
self.passthrough_violations = self.passthrough_violations.saturating_add(1);
let blocked = fatal || self.passthrough_violations > OPUS_PASSTHROUGH_STRIKE_LIMIT;
if blocked {
self.passthrough = Passthrough::Block;
}
blocked
}
}
impl Default for DecodeState {
fn default() -> Self {
Self {
inner_pos: 0,
resampler: None,
passthrough: Passthrough::Inactive,
passthrough_violations: 0,
}
}
}
/// Simple state to manage decoder resets etc.
///
/// Inactive->Active transitions should trigger a reset.
///
/// Block should be used if a source contains known-bad packets:
/// it's unlikely that packet sizes will vary, but if they do then
/// we can't passthrough (and every attempt will trigger a codec reset,
/// which probably won't sound too smooth).
#[derive(Clone, Copy, Eq, PartialEq)]
pub enum Passthrough {
Active,
Inactive,
Block,
}

View File

@@ -0,0 +1,403 @@
use crate::tracks::{ReadyState, SeekRequest};
use std::result::Result as StdResult;
use symphonia_core::errors::Error as SymphError;
use super::*;
pub struct InternalTrack {
pub(crate) playing: PlayMode,
pub(crate) volume: f32,
pub(crate) input: InputState,
pub(crate) mix_state: DecodeState,
pub(crate) position: Duration,
pub(crate) play_time: Duration,
pub(crate) commands: Receiver<TrackCommand>,
pub(crate) loops: LoopState,
pub(crate) callbacks: Callbacks,
}
impl<'a> InternalTrack {
pub(crate) fn decompose_track(
val: TrackContext,
) -> (Self, EventStore, TrackState, TrackHandle) {
let TrackContext {
handle,
track,
receiver,
} = val;
let out = InternalTrack {
playing: track.playing,
volume: track.volume,
input: InputState::from(track.input),
mix_state: DecodeState::default(),
position: Duration::default(),
play_time: Duration::default(),
commands: receiver,
loops: track.loops,
callbacks: Callbacks::default(),
};
let state = out.state();
(out, track.events, state, handle)
}
pub(crate) fn state(&self) -> TrackState {
let ready = self.input.ready_state();
TrackState {
playing: self.playing.clone(),
volume: self.volume,
position: self.position,
play_time: self.play_time,
loops: self.loops,
ready,
}
}
pub(crate) fn view(&'a mut self) -> View<'a> {
let ready = self.input.ready_state();
View {
position: &self.position,
play_time: &self.play_time,
volume: &mut self.volume,
meta: self.input.metadata(),
ready,
playing: &mut self.playing,
loops: &mut self.loops,
}
}
pub(crate) fn process_commands(&mut self, index: usize, ic: &Interconnect) -> Action {
// Note: disconnection and an empty channel are both valid,
// and should allow the audio object to keep running as intended.
// We also need to export a target seek point to the mixer, if known.
let mut action = Action::default();
// Note that interconnect failures are not currently errors.
// In correct operation, the event thread should never panic,
// but it receiving status updates is secondary do actually
// doing the work.
while let Ok(cmd) = self.commands.try_recv() {
match cmd {
TrackCommand::Play => {
self.playing.change_to(PlayMode::Play);
drop(ic.events.send(EventMessage::ChangeState(
index,
TrackStateChange::Mode(self.playing.clone()),
)));
},
TrackCommand::Pause => {
self.playing.change_to(PlayMode::Pause);
drop(ic.events.send(EventMessage::ChangeState(
index,
TrackStateChange::Mode(self.playing.clone()),
)));
},
TrackCommand::Stop => {
self.playing.change_to(PlayMode::Stop);
drop(ic.events.send(EventMessage::ChangeState(
index,
TrackStateChange::Mode(self.playing.clone()),
)));
},
TrackCommand::Volume(vol) => {
self.volume = vol;
drop(ic.events.send(EventMessage::ChangeState(
index,
TrackStateChange::Volume(self.volume),
)));
},
TrackCommand::Seek(req) => action.seek_point = Some(req),
TrackCommand::AddEvent(evt) => {
drop(ic.events.send(EventMessage::AddTrackEvent(index, evt)));
},
TrackCommand::Do(func) => {
if let Some(indiv_action) = func(self.view()) {
action.combine(indiv_action);
}
drop(ic.events.send(EventMessage::ChangeState(
index,
TrackStateChange::Total(self.state()),
)));
},
TrackCommand::Request(tx) => {
drop(tx.send(self.state()));
},
TrackCommand::Loop(loops) => {
self.loops = loops;
drop(ic.events.send(EventMessage::ChangeState(
index,
TrackStateChange::Loops(self.loops, true),
)));
},
TrackCommand::MakePlayable(callback) => action.make_playable = Some(callback),
}
}
action
}
pub(crate) fn do_loop(&mut self) -> bool {
match &mut self.loops {
LoopState::Infinite => true,
LoopState::Finite(n) =>
if let Some(new_n) = nonmax::NonMaxU32::new(n.get().wrapping_sub(1)) {
*n = new_n;
true
} else {
false
},
}
}
/// Steps playback location forward by one frame.
pub(crate) fn step_frame(&mut self) {
self.position += TIMESTEP_LENGTH;
self.play_time += TIMESTEP_LENGTH;
}
pub(crate) fn should_check_input(&self) -> bool {
self.playing.is_playing() || matches!(self.input, InputState::Preparing(_))
}
pub(crate) fn end(&mut self) -> &mut Self {
self.playing.change_to(PlayMode::End);
self
}
/// Readies the requested input state.
///
/// Returns the usable version of the audio if available, and whether the track should be deleted.
pub(crate) fn get_or_ready_input(
&'a mut self,
id: usize,
interconnect: &Interconnect,
pool: &BlockyTaskPool,
config: &Arc<Config>,
prevent_events: bool,
) -> StdResult<(&'a mut Parsed, &'a mut DecodeState), InputReadyingError> {
let input = &mut self.input;
let mix_state = &mut self.mix_state;
let (out, queued_seek) = match input {
InputState::NotReady(_) => {
let (tx, rx) = flume::bounded(1);
let mut state = InputState::Preparing(PreparingInfo {
time: Instant::now(),
queued_seek: None,
callback: rx,
});
std::mem::swap(&mut state, input);
match state {
InputState::NotReady(a @ Input::Lazy(_)) => {
pool.create(tx, a, None, config.clone());
},
InputState::NotReady(Input::Live(audio, rec)) => {
pool.parse(config.clone(), tx, audio, rec, None);
},
_ => unreachable!(),
}
if !prevent_events {
drop(interconnect.events.send(EventMessage::ChangeState(
id,
TrackStateChange::Ready(ReadyState::Preparing),
)));
}
(Err(InputReadyingError::Waiting), None)
},
InputState::Preparing(info) => {
let queued_seek = info.queued_seek.take();
let orig_out = match info.callback.try_recv() {
Ok(MixerInputResultMessage::Built(parsed, rec)) => {
*input = InputState::Ready(parsed, rec);
mix_state.reset();
// possible TODO: set position to the true track position here?
// ISSUE: need to get next_packet to see its `ts`, but inner_pos==0
// will trigger next packet to be taken at mix time.
if !prevent_events {
drop(interconnect.events.send(EventMessage::ChangeState(
id,
TrackStateChange::Ready(ReadyState::Playable),
)));
}
self.callbacks.playable();
if let InputState::Ready(ref mut parsed, _) = input {
Ok(parsed)
} else {
unreachable!()
}
},
Ok(MixerInputResultMessage::Seek(parsed, rec, seek_res)) => {
match seek_res {
Ok(pos) => {
if let Some(time_base) = parsed.decoder.codec_params().time_base {
// Update track's position to match the actual timestamp the
// seek landed at.
let new_time = time_base.calc_time(pos.actual_ts);
let time_in_float = new_time.seconds as f64 + new_time.frac;
self.position =
std::time::Duration::from_secs_f64(time_in_float);
self.callbacks.seeked(self.position);
self.callbacks.playable();
if !prevent_events {
drop(interconnect.events.send(EventMessage::ChangeState(
id,
TrackStateChange::Position(self.position),
)));
drop(interconnect.events.send(EventMessage::ChangeState(
id,
TrackStateChange::Ready(ReadyState::Playable),
)));
}
// Our decoder state etc. must be reset.
// (Symphonia decoder state reset in the thread pool during
// the operation.)
mix_state.reset();
*input = InputState::Ready(parsed, rec);
if let InputState::Ready(ref mut parsed, _) = input {
Ok(parsed)
} else {
unreachable!()
}
} else {
Err(InputReadyingError::Seeking(
SymphError::Unsupported("Track had no recorded time base.")
.into(),
))
}
},
Err(e) => Err(InputReadyingError::Seeking(e)),
}
},
Ok(MixerInputResultMessage::CreateErr(e)) =>
Err(InputReadyingError::Creation(e)),
Ok(MixerInputResultMessage::ParseErr(e)) => Err(InputReadyingError::Parsing(e)),
Err(TryRecvError::Disconnected) => Err(InputReadyingError::Dropped),
Err(TryRecvError::Empty) => Err(InputReadyingError::Waiting),
};
let orig_out = orig_out.map(|a| (a, mix_state));
if let Err(ref e) = orig_out {
if let Some(e) = e.as_user() {
self.callbacks.readying_error(e);
}
}
(orig_out, queued_seek)
},
InputState::Ready(ref mut parsed, _) => (Ok((parsed, mix_state)), None),
};
match (out, queued_seek) {
(Ok(_), Some(request)) => Err(InputReadyingError::NeedsSeek(request)),
(a, _) => a,
}
}
pub(crate) fn seek(
&mut self,
id: usize,
request: SeekRequest,
interconnect: &Interconnect,
pool: &BlockyTaskPool,
config: &Arc<Config>,
prevent_events: bool,
) {
if let InputState::Preparing(p) = &mut self.input {
p.queued_seek = Some(request);
return;
}
// might be a little topsy turvy: rethink me.
let SeekRequest { time, callback } = request;
self.callbacks.seek = Some(callback);
if !prevent_events {
drop(interconnect.events.send(EventMessage::ChangeState(
id,
TrackStateChange::Ready(ReadyState::Preparing),
)));
}
let backseek_needed = time < self.position;
let time = Time::from(time.as_secs_f64());
let mut ts = SeekTo::Time {
time,
track_id: None,
};
let (tx, rx) = flume::bounded(1);
let state = std::mem::replace(
&mut self.input,
InputState::Preparing(PreparingInfo {
time: Instant::now(),
callback: rx,
queued_seek: None,
}),
);
match state {
InputState::Ready(p, r) => {
if let SeekTo::Time { time: _, track_id } = &mut ts {
*track_id = Some(p.track_id);
}
pool.seek(tx, p, r, ts, backseek_needed, config.clone());
},
InputState::NotReady(lazy) => pool.create(tx, lazy, Some(ts), config.clone()),
InputState::Preparing(_) => unreachable!(), // Covered above.
}
}
}
#[derive(Debug, Default)]
pub struct Callbacks {
pub seek: Option<Sender<StdResult<Duration, PlayError>>>,
pub make_playable: Option<Sender<StdResult<(), PlayError>>>,
}
impl Callbacks {
fn readying_error(&mut self, err: PlayError) {
if let Some(callback) = self.seek.take() {
drop(callback.send(Err(err.clone())));
}
if let Some(callback) = self.make_playable.take() {
drop(callback.send(Err(err)));
}
}
fn playable(&mut self) {
if let Some(callback) = self.make_playable.take() {
drop(callback.send(Ok(())));
}
}
fn seeked(&mut self, time: Duration) {
if let Some(callback) = self.seek.take() {
drop(callback.send(Ok(time)));
}
}
}

View File

@@ -0,0 +1,20 @@
use symphonia_core::{formats::SeekTo, units::Time};
// SeekTo lacks Copy and Clone... somehow.
pub fn copy_seek_to(pos: &SeekTo) -> SeekTo {
match *pos {
SeekTo::Time { time, track_id } => SeekTo::Time { time, track_id },
SeekTo::TimeStamp { ts, track_id } => SeekTo::TimeStamp { ts, track_id },
}
}
pub fn seek_to_is_zero(pos: &SeekTo) -> bool {
match *pos {
SeekTo::Time { time, .. } =>
time == Time {
seconds: 0,
frac: 0.0,
},
SeekTo::TimeStamp { ts, .. } => ts == 0,
}
}

View File

@@ -0,0 +1,351 @@
#![allow(missing_docs)]
pub(crate) mod disposal;
pub mod error;
mod events;
pub mod message;
pub mod mixer;
#[cfg(feature = "receive")]
pub(crate) mod udp_rx;
pub(crate) mod ws;
use super::connection::{error::Error as ConnectionError, Connection};
use crate::{
events::{
context_data::{DisconnectKind, DisconnectReason},
internal_data::{InternalConnect, InternalDisconnect},
CoreContext,
},
Config,
ConnectionInfo,
FloatDuration,
};
use flume::{Receiver, Sender};
use message::*;
use tokio::{spawn, time::sleep as tsleep};
use tracing::{debug, instrument, trace};
pub(crate) fn start(config: Config, rx: Receiver<CoreMessage>, tx: Sender<CoreMessage>) {
spawn(async move {
trace!("Driver started.");
runner(config, rx, tx).await;
trace!("Driver finished.");
});
}
fn start_internals(core: Sender<CoreMessage>, config: &Config) -> Interconnect {
let (evt_tx, evt_rx) = flume::unbounded();
let (mix_tx, mix_rx) = flume::unbounded();
spawn(async move {
trace!("Event processor started.");
events::runner(evt_rx).await;
trace!("Event processor finished.");
});
let ic = Interconnect {
core,
events: evt_tx,
mixer: mix_tx,
};
config.get_scheduler().new_mixer(config, ic.clone(), mix_rx);
ic
}
#[instrument(skip(rx, tx))]
async fn runner(mut config: Config, rx: Receiver<CoreMessage>, tx: Sender<CoreMessage>) {
let mut next_config: Option<Config> = None;
let mut connection: Option<Connection> = None;
let mut interconnect = start_internals(tx, &config);
let mut retrying = None;
let mut attempt_idx = 0;
while let Ok(msg) = rx.recv_async().await {
match msg {
CoreMessage::ConnectWithResult(info, tx) => {
config = if let Some(new_config) = next_config.take() {
drop(
interconnect
.mixer
.send(MixerMessage::SetConfig(new_config.clone())),
);
new_config
} else {
config
};
if connection.as_ref().is_none_or(|conn| conn.info != info) {
// Only *actually* reconnect if the conn info changed, or we don't have an
// active connection.
// This allows the gateway component to keep sending join requests independent
// of driver failures.
connection = ConnectionRetryData::connect(tx, info, &mut attempt_idx)
.attempt(&mut retrying, &interconnect, &config)
.await;
} else {
// No reconnection was attempted as there's a valid, identical connection;
// tell the outside listener that the operation was a success.
drop(tx.send(Ok(())));
}
},
CoreMessage::RetryConnect(retry_idx) => {
debug!("Retrying idx: {} (vs. {})", retry_idx, attempt_idx);
if retry_idx == attempt_idx {
if let Some(progress) = retrying.take() {
connection = progress
.attempt(&mut retrying, &interconnect, &config)
.await;
}
}
},
CoreMessage::Disconnect => {
let last_conn = connection.take();
drop(interconnect.mixer.send(MixerMessage::DropConn));
drop(interconnect.mixer.send(MixerMessage::RebuildEncoder));
if let Some(conn) = last_conn {
drop(interconnect.events.send(EventMessage::FireCoreEvent(
CoreContext::DriverDisconnect(InternalDisconnect {
kind: DisconnectKind::Runtime,
reason: Some(DisconnectReason::Requested),
info: conn.info.clone(),
}),
)));
}
},
CoreMessage::SignalWsClosure(ws_idx, ws_info, mut reason) => {
// if idx is not a match, quash reason
// (i.e., prevent users from mistakenly trying to reconnect for an *old* dead conn).
// if it *is* a match, the conn needs to die!
// (as the WS channel has truly given up the ghost).
let conn = if ws_idx == attempt_idx {
drop(interconnect.mixer.send(MixerMessage::DropConn));
drop(interconnect.mixer.send(MixerMessage::RebuildEncoder));
connection.take()
} else {
reason = None;
None
};
// Conn may have been unset earlier (i.e., in a deliberate disconnect).
// If so, do not repropagate/repeat the disconnect event.
if conn.is_some() {
drop(interconnect.events.send(EventMessage::FireCoreEvent(
CoreContext::DriverDisconnect(InternalDisconnect {
kind: DisconnectKind::Runtime,
reason,
info: ws_info,
}),
)));
}
},
CoreMessage::SetTrack(s) => {
drop(interconnect.mixer.send(MixerMessage::SetTrack(s)));
},
CoreMessage::AddTrack(s) => {
drop(interconnect.mixer.send(MixerMessage::AddTrack(s)));
},
CoreMessage::SetBitrate(b) => {
drop(interconnect.mixer.send(MixerMessage::SetBitrate(b)));
},
CoreMessage::SetConfig(mut new_config) => {
next_config = Some(new_config.clone());
new_config.make_safe(&config, connection.is_some());
drop(interconnect.mixer.send(MixerMessage::SetConfig(new_config)));
},
CoreMessage::AddEvent(evt) => {
drop(interconnect.events.send(EventMessage::AddGlobalEvent(evt)));
},
CoreMessage::RemoveGlobalEvents => {
drop(interconnect.events.send(EventMessage::RemoveGlobalEvents));
},
CoreMessage::Mute(m) => {
drop(interconnect.mixer.send(MixerMessage::SetMute(m)));
},
CoreMessage::Reconnect => {
if let Some(mut conn) = connection.take() {
// try once: if interconnect, try again.
// if still issue, full connect.
let info = conn.info.clone();
let full_connect = match conn.reconnect(&config).await {
Ok(()) => {
connection = Some(conn);
false
},
Err(ConnectionError::InterconnectFailure(_)) => {
interconnect.restart_volatile_internals();
match conn.reconnect(&config).await {
Ok(()) => {
connection = Some(conn);
false
},
_ => true,
}
},
_ => true,
};
if full_connect {
connection = ConnectionRetryData::reconnect(info, &mut attempt_idx)
.attempt(&mut retrying, &interconnect, &config)
.await;
} else if let Some(ref connection) = &connection {
drop(interconnect.events.send(EventMessage::FireCoreEvent(
CoreContext::DriverReconnect(InternalConnect {
info: connection.info.clone(),
ssrc: connection.ssrc,
}),
)));
}
}
},
CoreMessage::FullReconnect =>
if let Some(conn) = connection.take() {
let info = conn.info.clone();
connection = ConnectionRetryData::reconnect(info, &mut attempt_idx)
.attempt(&mut retrying, &interconnect, &config)
.await;
},
CoreMessage::RebuildInterconnect => {
interconnect.restart_volatile_internals();
},
CoreMessage::Poison => break,
}
}
trace!("Main thread exited");
interconnect.poison_all();
}
struct ConnectionRetryData {
flavour: ConnectionFlavour,
attempts: u8,
last_wait: Option<FloatDuration>,
info: ConnectionInfo,
idx: usize,
}
impl ConnectionRetryData {
fn connect(
tx: Sender<Result<(), ConnectionError>>,
info: ConnectionInfo,
idx_src: &mut usize,
) -> Self {
Self::base(ConnectionFlavour::Connect(tx), info, idx_src)
}
fn reconnect(info: ConnectionInfo, idx_src: &mut usize) -> Self {
Self::base(ConnectionFlavour::Reconnect, info, idx_src)
}
fn base(flavour: ConnectionFlavour, info: ConnectionInfo, idx_src: &mut usize) -> Self {
*idx_src = idx_src.wrapping_add(1);
Self {
flavour,
attempts: 0,
last_wait: None,
info,
idx: *idx_src,
}
}
async fn attempt(
mut self,
attempt_slot: &mut Option<Self>,
interconnect: &Interconnect,
config: &Config,
) -> Option<Connection> {
match Connection::new(self.info.clone(), interconnect, config, self.idx).await {
Ok(connection) => {
match self.flavour {
ConnectionFlavour::Connect(tx) => {
// Other side may not be listening: this is fine.
drop(tx.send(Ok(())));
drop(interconnect.events.send(EventMessage::FireCoreEvent(
CoreContext::DriverConnect(InternalConnect {
info: connection.info.clone(),
ssrc: connection.ssrc,
}),
)));
},
ConnectionFlavour::Reconnect => {
drop(interconnect.events.send(EventMessage::FireCoreEvent(
CoreContext::DriverReconnect(InternalConnect {
info: connection.info.clone(),
ssrc: connection.ssrc,
}),
)));
},
}
Some(connection)
},
Err(why) => {
debug!("Failed to connect for {:?}: {}", self.info.guild_id, why);
if let Some(t) = config.driver_retry.retry_in(self.last_wait, self.attempts) {
let remote_ic = interconnect.clone();
let idx = self.idx;
spawn(async move {
tsleep(t.into()).await;
drop(remote_ic.core.send(CoreMessage::RetryConnect(idx)));
});
self.attempts += 1;
self.last_wait = Some(t);
debug!(
"Retrying connection for {:?} in {}s ({}/{:?})",
self.info.guild_id,
t.as_secs_f32(),
self.attempts,
config.driver_retry.retry_limit
);
*attempt_slot = Some(self);
} else {
let reason = Some(DisconnectReason::from(&why));
match self.flavour {
ConnectionFlavour::Connect(tx) => {
// See above.
drop(tx.send(Err(why)));
drop(interconnect.events.send(EventMessage::FireCoreEvent(
CoreContext::DriverDisconnect(InternalDisconnect {
kind: DisconnectKind::Connect,
reason,
info: self.info,
}),
)));
},
ConnectionFlavour::Reconnect => {
drop(interconnect.events.send(EventMessage::FireCoreEvent(
CoreContext::DriverDisconnect(InternalDisconnect {
kind: DisconnectKind::Reconnect,
reason,
info: self.info,
}),
)));
},
}
}
None
},
}
}
}
enum ConnectionFlavour {
Connect(Sender<Result<(), ConnectionError>>),
Reconnect,
}

View File

@@ -0,0 +1,42 @@
use crate::constants::STEREO_FRAME_SIZE;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum PacketDecodeSize {
/// Minimum frame size on Discord.
TwentyMillis,
/// Hybrid packet, sent by Firefox web client.
///
/// Likely 20ms frame + 10ms frame.
ThirtyMillis,
/// Next largest frame size.
FortyMillis,
/// Maximum Opus frame size.
SixtyMillis,
/// Maximum Opus packet size: 120ms.
Max,
}
impl PacketDecodeSize {
pub fn bump_up(self) -> Self {
match self {
Self::TwentyMillis => Self::ThirtyMillis,
Self::ThirtyMillis => Self::FortyMillis,
Self::FortyMillis => Self::SixtyMillis,
Self::SixtyMillis | Self::Max => Self::Max,
}
}
pub fn can_bump_up(self) -> bool {
self != Self::Max
}
pub fn len(self) -> usize {
match self {
Self::TwentyMillis => STEREO_FRAME_SIZE,
Self::ThirtyMillis => (STEREO_FRAME_SIZE / 2) * 3,
Self::FortyMillis => 2 * STEREO_FRAME_SIZE,
Self::SixtyMillis => 3 * STEREO_FRAME_SIZE,
Self::Max => 6 * STEREO_FRAME_SIZE,
}
}
}

View File

@@ -0,0 +1,114 @@
//! Jarvis patch: RTP packets whose SSRC Discord has not named yet are held for a moment.
//!
//! Discord names a speaker's SSRC in a Speaking op over the voice websocket the first time they speak,
//! and that op arrives after their first packets over UDP. Unpatched, songbird dropped DAVE packets
//! of an unnamed SSRC (it needs the user to decrypt them), which cut the first ~100 ms off the first
//! thing each person said after a join ("자비스" heard as "아비"). Held packets are processed in
//! arrival order before the first packet after the name arrives.
use bytes::BytesMut;
use std::collections::{HashMap, VecDeque};
use std::time::Duration;
use tokio::time::Instant;
/// A packet older than this when its SSRC is named is dropped.
pub(crate) const HOLD: Duration = Duration::from_secs(1);
/// At most this many packets per SSRC: one second of 20 ms packets.
pub(crate) const MAX_PACKETS: usize = 50;
/// At most this many unnamed SSRCs at once.
pub(crate) const MAX_SSRCS: usize = 32;
#[derive(Default)]
pub(crate) struct Held {
by_ssrc: HashMap<u32, VecDeque<(Instant, BytesMut)>>,
}
impl Held {
pub(crate) fn hold(&mut self, ssrc: u32, at: Instant, packet: BytesMut) {
if !self.by_ssrc.contains_key(&ssrc) && self.by_ssrc.len() >= MAX_SSRCS {
return;
}
let queue = self.by_ssrc.entry(ssrc).or_default();
while queue.front().is_some_and(|(t, _)| at.duration_since(*t) > HOLD) || queue.len() >= MAX_PACKETS {
queue.pop_front();
}
queue.push_back((at, packet));
}
/// The packets held for `ssrc` that are at most `HOLD` old, oldest first. Forgets all of them.
pub(crate) fn take(&mut self, ssrc: u32, now: Instant) -> Vec<BytesMut> {
self.by_ssrc
.remove(&ssrc)
.map(|q| q.into_iter().filter(|(t, _)| now.duration_since(*t) <= HOLD).map(|(_, p)| p).collect())
.unwrap_or_default()
}
/// Forgets every SSRC whose newest packet is older than `HOLD`.
pub(crate) fn prune(&mut self, now: Instant) {
self.by_ssrc.retain(|_, q| q.back().is_some_and(|(t, _)| now.duration_since(*t) <= HOLD));
}
}
#[cfg(test)]
mod tests {
use super::*;
fn p(n: u8) -> BytesMut {
BytesMut::from(&[n][..])
}
#[test]
fn packets_held_before_the_name_come_back_in_arrival_order() {
let mut h = Held::default();
let t = Instant::now();
for n in 0..3 {
h.hold(7, t + Duration::from_millis(20 * u64::from(n)), p(n));
}
h.hold(8, t, p(9));
assert_eq!(h.take(7, t + Duration::from_millis(100)), vec![p(0), p(1), p(2)]);
assert!(h.take(7, t + Duration::from_millis(100)).is_empty(), "taken once");
assert_eq!(h.take(8, t), vec![p(9)], "other speakers are kept apart");
}
#[test]
fn packets_older_than_the_hold_are_dropped() {
let mut h = Held::default();
let t = Instant::now();
h.hold(7, t, p(0));
h.hold(7, t + Duration::from_millis(900), p(1));
assert_eq!(h.take(7, t + Duration::from_millis(1500)), vec![p(1)]);
h.hold(7, t, p(0));
h.hold(7, t + HOLD + Duration::from_millis(20), p(1));
assert_eq!(h.take(7, t + HOLD + Duration::from_millis(20)), vec![p(1)], "dropped as newer ones come");
}
#[test]
fn at_most_a_second_of_packets_per_speaker_and_a_bounded_number_of_speakers() {
let mut h = Held::default();
let t = Instant::now();
for n in 0..(MAX_PACKETS as u8 + 5) {
h.hold(7, t, p(n));
}
let kept = h.take(7, t);
assert_eq!(kept.len(), MAX_PACKETS);
assert_eq!(kept[0], p(5), "the newest are kept");
for ssrc in 0..(MAX_SSRCS as u32 + 3) {
h.hold(100 + ssrc, t, p(1));
}
assert!(h.take(100 + MAX_SSRCS as u32, t).is_empty());
assert_eq!(h.take(100, t), vec![p(1)]);
}
#[test]
fn prune_forgets_speakers_that_stopped_sending() {
let mut h = Held::default();
let t = Instant::now();
h.hold(7, t, p(0));
h.hold(8, t + Duration::from_millis(1500), p(1));
h.prune(t + Duration::from_millis(1600));
assert!(h.take(7, t).is_empty());
assert_eq!(h.take(8, t + Duration::from_millis(1600)), vec![p(1)]);
}
}

View File

@@ -0,0 +1,442 @@
mod decode_sizes;
mod held;
mod playout_buffer;
mod ssrc_state;
use self::{decode_sizes::*, held::Held, playout_buffer::*, ssrc_state::*};
use super::message::*;
use crate::driver::{CryptoMode, DecodeMode};
use crate::{
constants::*,
driver::crypto::Cipher,
events::{context_data::VoiceTick, internal_data::*, CoreContext},
Config,
};
use bytes::BytesMut;
use discortp::{
demux::{self, DemuxedMut},
rtp::RtpPacket,
};
use discortp::{MutablePacket, Packet};
use flume::Receiver;
use std::sync::{
atomic::{AtomicU16, Ordering},
RwLock,
};
use std::{
collections::{HashMap, HashSet},
num::Wrapping,
sync::Arc,
time::Duration,
};
use tokio::{net::UdpSocket, select, time::Instant};
use tracing::{error, instrument, trace, warn};
type RtpSequence = Wrapping<u16>;
type RtpTimestamp = Wrapping<u32>;
type RtpSsrc = u32;
struct UdpRx {
cipher: Cipher,
crypto_mode: CryptoMode,
decoder_map: HashMap<RtpSsrc, SsrcState>,
config: Config,
rx: Receiver<UdpRxMessage>,
ssrc_signalling: Arc<SsrcTracker>,
udp_socket: UdpSocket,
dave_session: Arc<RwLock<Option<davey::DaveSession>>>,
dave_protocol_version: Arc<AtomicU16>,
/// Jarvis patch: packets of SSRCs not named yet (see held.rs).
held: Held,
}
impl UdpRx {
#[instrument(skip(self))]
async fn run(&mut self, interconnect: &mut Interconnect) {
let mut cleanup_time = Instant::now();
let mut playout_time = Instant::now() + TIMESTEP_LENGTH;
let mut byte_dest: Option<BytesMut> = None;
loop {
if byte_dest.is_none() {
byte_dest = Some(BytesMut::zeroed(VOICE_PACKET_MAX));
}
select! {
Ok((len, _addr)) = self.udp_socket.recv_from(byte_dest.as_mut().unwrap()) => {
let mut pkt = byte_dest.take().unwrap();
pkt.truncate(len);
self.process_udp_message(interconnect, pkt).await;
},
msg = self.rx.recv_async() => {
match msg {
Ok(UdpRxMessage::ReplaceInterconnect(i)) => {
*interconnect = i;
},
Ok(UdpRxMessage::SetConfig(new_config)) => {
if let DecodeMode::Decode(old_config) = &mut self.config.decode_mode {
if *old_config != new_config {
*old_config = new_config;
self.decoder_map.values_mut().for_each(|v| v.reconfigure_decoder(new_config));
}
}
},
Err(flume::RecvError::Disconnected) => break,
}
},
() = tokio::time::sleep_until(playout_time) => {
let mut tick = VoiceTick {
speaking: HashMap::new(),
silent: HashSet::new(),
};
for (ssrc, state) in &mut self.decoder_map {
match state.get_voice_tick(&self.config) {
Ok(Some(data)) => {
tick.speaking.insert(*ssrc, data);
},
Ok(None) => {
if !state.disconnected {
tick.silent.insert(*ssrc);
}
},
Err(e) => {
warn!("Decode error for SSRC {ssrc}: {e:?}");
tick.silent.insert(*ssrc);
},
}
}
playout_time += TIMESTEP_LENGTH;
drop(interconnect.events.send(EventMessage::FireCoreEvent(CoreContext::VoiceTick(tick))));
},
() = tokio::time::sleep_until(cleanup_time) => {
// periodic cleanup.
let now = Instant::now();
// check ssrc map to see if the WS task has informed us of any disconnects.
loop {
// This is structured in an odd way to prevent deadlocks.
// while-let seemed to keep the dashmap iter() alive for block scope, rather than
// just the initialiser.
let id = {
if let Some(id) = self.ssrc_signalling.disconnected_users.iter().next().map(|v| *v.key()) {
id
} else {
break;
}
};
_ = self.ssrc_signalling.disconnected_users.remove(&id);
if let Some((_, ssrc)) = self.ssrc_signalling.user_ssrc_map.remove(&id) {
let _ = self.ssrc_signalling.ssrc_user_map.remove(&ssrc);
if let Some(state) = self.decoder_map.get_mut(&ssrc) {
// don't cleanup immediately: leave for later cycle
// this is key with reorder/jitter buffers where we may
// still need to decode post disconnect for ~0.2s.
state.prune_time = now + Duration::from_secs(1);
state.disconnected = true;
}
}
}
// now remove all dead ssrcs.
self.decoder_map.retain(|_, v| v.prune_time > now);
self.held.prune(now);
cleanup_time = now + Duration::from_secs(5);
},
}
}
}
/// Jarvis patch: an RTP packet of an SSRC that Discord has not named yet is held, and the held
/// ones are processed before the first packet after the name came (see held.rs).
async fn process_udp_message(&mut self, interconnect: &Interconnect, packet: BytesMut) {
if let demux::Demuxed::Rtp(rtp) = demux::demux(&packet) {
let ssrc = rtp.get_ssrc();
let now = Instant::now();
if !self.ssrc_signalling.ssrc_user_map.contains_key(&ssrc) {
self.held.hold(ssrc, now, packet);
return;
}
for held in self.held.take(ssrc, now) {
self.process_packet(interconnect, held).await;
}
}
self.process_packet(interconnect, packet).await;
}
async fn process_packet(&mut self, interconnect: &Interconnect, mut packet: BytesMut) {
// NOTE: errors here (and in general for UDP) are not fatal to the connection.
// Panics should be avoided due to adversarial nature of rx'd packets,
// but correct handling should not prompt a reconnect.
//
// For simplicity, if the event task fails then we nominate the mixing thread
// to rebuild their context etc. (hence, the `let _ =` statements.), as it will
// try to make contact every 20ms.
let crypto_mode = self.crypto_mode;
match demux::demux_mut(packet.as_mut()) {
DemuxedMut::Rtp(mut rtp) => {
if !rtp_valid(&rtp.to_immutable()) {
error!("Illegal RTP message received.");
return;
}
let mut packet_data = if self.config.decode_mode.should_decrypt() {
let out = self.cipher.decrypt_rtp_in_place(&mut rtp).map(|(s, t)| {
if rtp.get_padding() != 0 {
let payload = rtp.payload();
let payload_length = payload.len();
let padding_count = payload[payload_length - t - 1] as usize;
(s, t + padding_count, true)
} else {
(s, t, true)
}
});
if let Err(ref e) = out {
warn!("RTP decryption failed: {:?}", e);
}
out.ok()
} else {
None
};
if let Some((rtp_body_start, rtp_body_tail, decrypted)) = packet_data {
let ssrc = rtp.get_ssrc();
let payload = rtp.payload_mut();
let payload_length = payload.len();
let body = &mut payload[rtp_body_start..payload_length - rtp_body_tail];
let body_length = body.len();
// If the packet is transport-decrypted, DAVE is enabled, and the packet
// looks encrypted: https://daveprotocol.com/#protocol-frame-check
// The packet must be at least 11 bytes consisting of:
// - 8 byte truncated AES128-GCM authentication tag
// - 1 byte protocol supplemental data size
// - 2 byte magic marker
if decrypted
&& self.dave_protocol_version.load(Ordering::Relaxed) != 0
&& body_length >= 11
&& body[body_length - DAVE_MAGIC_MARKER.len()..body_length]
== DAVE_MAGIC_MARKER
{
// Silently drop encrypted packets if it's not possible to decrypt them for
// one reason or another; otherwise there'd be error logs for trying to decode
// Opus
let Some(user_id) = self.ssrc_signalling.ssrc_user_map.get(&ssrc) else {
return;
};
let Some(ref mut dave_session) = *self.dave_session.write().unwrap() else {
return;
};
if !dave_session.is_ready() {
return;
}
let result = dave_session.decrypt(user_id.0, davey::MediaType::AUDIO, body);
match result {
Ok(decrypted_body) => {
packet_data = Some((
rtp_body_start,
rtp_body_tail + (body.len() - decrypted_body.len()),
decrypted,
));
body[..decrypted_body.len()].copy_from_slice(&decrypted_body);
},
Err(davey::errors::DecryptError::NoDecryptorForUser)
| Err(davey::errors::DecryptError::DecryptionFailed(
davey::errors::DecryptorDecryptError::NoValidCryptorFound {
..
},
)) => {
// Silently drop encrypted packets for users whose ratchets are not configured yet.
return;
},
Err(e) => {
error!(error = ?e, "DAVE decryption failed");
return;
},
}
}
}
let rtp = rtp.to_immutable();
let (rtp_body_start, rtp_body_tail, decrypted) = packet_data.unwrap_or_else(|| {
(
crypto_mode.payload_prefix_len(),
crypto_mode.payload_suffix_len(),
false,
)
});
let entry = self
.decoder_map
.entry(rtp.get_ssrc())
.or_insert_with(|| SsrcState::new(&rtp, crypto_mode, &self.config));
// Only do this on RTP, rather than RTCP -- this pins decoder state liveness
// to *speech* rather than just presence.
entry.refresh_timer(self.config.decode_state_timeout.into());
let store_pkt = StoredPacket {
packet: packet.freeze(),
decrypted,
};
let packet = store_pkt.packet.clone();
entry.store_packet(store_pkt, &self.config);
drop(interconnect.events.send(EventMessage::FireCoreEvent(
CoreContext::RtpPacket(InternalRtpPacket {
packet,
payload_offset: rtp_body_start,
payload_end_pad: rtp_body_tail,
}),
)));
},
DemuxedMut::Rtcp(mut rtcp) => {
let packet_data = if self.config.decode_mode.should_decrypt() {
let out = self.cipher.decrypt_rtcp_in_place(&mut rtcp);
if let Err(ref e) = out {
warn!("RTCP decryption failed: {:?}", e);
}
out.ok()
} else {
None
};
let (start, tail) = packet_data.unwrap_or_else(|| {
(
crypto_mode.payload_prefix_len(),
crypto_mode.payload_suffix_len(),
)
});
drop(interconnect.events.send(EventMessage::FireCoreEvent(
CoreContext::RtcpPacket(InternalRtcpPacket {
packet: packet.freeze(),
payload_offset: start,
payload_end_pad: tail,
}),
)));
},
DemuxedMut::FailedParse(t) => {
warn!("Failed to parse message of type {:?}.", t);
},
DemuxedMut::TooSmall => {
warn!("Illegal UDP packet from voice server.");
},
}
}
}
#[instrument(skip(interconnect, rx, cipher))]
pub(crate) async fn runner(
mut interconnect: Interconnect,
rx: Receiver<UdpRxMessage>,
cipher: Cipher,
crypto_mode: CryptoMode,
config: Config,
udp_socket: UdpSocket,
ssrc_signalling: Arc<SsrcTracker>,
dave_session: Arc<RwLock<Option<davey::DaveSession>>>,
dave_protocol_version: Arc<AtomicU16>,
) {
trace!("UDP receive handle started.");
let mut state = UdpRx {
cipher,
crypto_mode,
decoder_map: HashMap::new(),
config,
rx,
ssrc_signalling,
udp_socket,
dave_session,
dave_protocol_version,
held: Held::default(),
};
state.run(&mut interconnect).await;
trace!("UDP receive handle stopped.");
}
#[inline]
fn rtp_valid(packet: &RtpPacket<'_>) -> bool {
packet.get_version() == RTP_VERSION && packet.get_payload_type() == RTP_PROFILE_TYPE
}
#[cfg(test)]
mod jarvis_held_tests {
//! Jarvis patch (held.rs): the first packets of a speaker are not lost to a late Speaking op.
use super::*;
use crate::driver::DecodeMode;
use discortp::rtp::MutableRtpPacket;
use serenity_voice_model::id::UserId;
fn rtp(seq: u16, ssrc: u32) -> BytesMut {
let mut buf = BytesMut::zeroed(MutableRtpPacket::minimum_packet_size() + 3);
let mut p = MutableRtpPacket::new(&mut buf[..]).unwrap();
p.set_version(RTP_VERSION);
p.set_payload_type(RTP_PROFILE_TYPE);
p.set_sequence(seq.into());
p.set_ssrc(ssrc);
buf
}
fn sequences(events: &Receiver<EventMessage>) -> Vec<u16> {
events
.try_iter()
.filter_map(|e| match e {
EventMessage::FireCoreEvent(CoreContext::RtpPacket(p)) =>
Some(RtpPacket::new(&p.packet).unwrap().get_sequence().0 .0),
_ => None,
})
.collect()
}
#[tokio::test]
async fn packets_before_the_speaking_op_reach_handlers_once_it_names_the_ssrc() {
let (core, _core_rx) = flume::unbounded();
let (events, events_rx) = flume::unbounded();
let (mixer, _mixer_rx) = flume::unbounded();
let ic = Interconnect { core, events, mixer };
let mode = CryptoMode::Aes256Gcm;
let mut rx = UdpRx {
cipher: mode.cipher_from_key(&[0u8; 32]).unwrap(),
crypto_mode: mode,
decoder_map: HashMap::new(),
config: Config::default().decode_mode(DecodeMode::Pass),
rx: flume::unbounded().1,
ssrc_signalling: Arc::new(SsrcTracker::default()),
udp_socket: UdpSocket::bind("127.0.0.1:0").await.unwrap(),
dave_session: Arc::new(RwLock::new(None)),
dave_protocol_version: Arc::new(AtomicU16::new(0)),
held: Held::default(),
};
rx.process_udp_message(&ic, rtp(1, 42)).await;
rx.process_udp_message(&ic, rtp(2, 42)).await;
assert!(sequences(&events_rx).is_empty(), "held until the SSRC is named");
rx.ssrc_signalling.ssrc_user_map.insert(42, UserId(7));
rx.process_udp_message(&ic, rtp(3, 42)).await;
assert_eq!(sequences(&events_rx), vec![1, 2, 3]);
rx.process_udp_message(&ic, rtp(4, 42)).await;
assert_eq!(sequences(&events_rx), vec![4], "a held packet is processed once");
}
}

View File

@@ -0,0 +1,204 @@
use super::*;
use bytes::Bytes;
use std::collections::VecDeque;
use tracing::trace;
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct StoredPacket {
pub packet: Bytes,
// We need to store this as it's possible that a user can change config modes.
pub decrypted: bool,
}
/// Determines whether an SSRC's packets should be decoded.
///
/// Playout requires us to keep an almost constant delay, to do so we build
/// a user's packet buffer up to the required length ([`Config::playout_buffer_length`])
/// ([`Self::Fill`]) and then emit packets on each tick ([`Self::Drain`]).
///
/// This gets a bit harder to reason about when users stop speaking. If a speech gap
/// lasts longer than the playout buffer, then we can simply swap from `Drain` -> `Fill`.
/// However, a genuine gap of `n` frames must lead to us reverting to `Fill` for `n` frames.
/// To compute this, we use the RTP timestamp of two `seq`-adjacent packets at playout: if the next
/// timestamp is too large, then we revert to `Fill`.
///
/// Small playout bursts also require care.
///
/// If timestamp info is incorrect, then in the worst case we eventually need to rebuffer if the delay
/// drains to zero.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum PlayoutMode {
Fill,
Drain,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum PacketLookup {
Packet(StoredPacket),
MissedPacket,
Filling,
}
#[derive(Debug)]
pub struct PlayoutBuffer {
buffer: VecDeque<Option<StoredPacket>>,
playout_mode: PlayoutMode,
next_seq: RtpSequence,
current_timestamp: Option<RtpTimestamp>,
consecutive_store_fails: usize,
}
impl PlayoutBuffer {
pub fn new(capacity: usize, next_seq: RtpSequence) -> Self {
Self {
buffer: VecDeque::with_capacity(capacity),
playout_mode: PlayoutMode::Fill,
next_seq,
current_timestamp: None,
consecutive_store_fails: 0,
}
}
/// Slot a received RTP packet into the correct location in the playout buffer using
/// its sequence number, subject to maximums.
///
/// An out of bounds packet must create any remaining `None`s
pub fn store_packet(&mut self, packet: StoredPacket, config: &Config) {
let rtp = RtpPacket::new(&packet.packet)
.expect("FATAL: earlier valid packet now invalid (store)");
if self.current_timestamp.is_none() {
self.current_timestamp = Some(reset_timeout(&rtp, config));
}
// compute index by taking wrapping difference between both seq numbers.
// If the difference is *too big*, or in the past [also too big, in a way],
// ignore the packet
let pkt_seq = rtp.get_sequence().0;
let desired_index = (pkt_seq - self.next_seq).0 as i16;
// Similar concept to fetch_packet -- if there's a critical desync, and we're unwilling
// to slot this packet into an empty/stuck buffer then behave as though this packet is the next
// sequence number we're releasing.
let err_threshold = i16::from(config.playout_buffer_length.get() * 5);
let handling_desync = (self.buffer.is_empty()
|| self.consecutive_store_fails >= (err_threshold as usize))
&& desired_index >= err_threshold;
if desired_index < 0 {
trace!("Missed packet arrived late, discarding from playout.");
} else if !handling_desync && desired_index >= 64 {
trace!(
"Packet arrived beyond playout max length({}): wanted slot {desired_index}.\
ts {}, seq {}, next_out_seq {}",
rtp.get_ssrc(),
rtp.get_timestamp().0,
rtp.get_sequence().0,
self.next_seq,
);
self.consecutive_store_fails += 1;
} else {
let index = if handling_desync {
self.buffer.clear();
self.next_seq = pkt_seq;
0
} else {
desired_index as usize
};
while self.buffer.len() <= index {
self.buffer.push_back(None);
}
self.buffer[index] = Some(packet);
self.consecutive_store_fails = 0;
}
if self.buffer.len() >= usize::from(config.playout_buffer_length.get()) {
self.playout_mode = PlayoutMode::Drain;
}
}
pub fn fetch_packet(&mut self, config: &Config) -> PacketLookup {
if self.playout_mode == PlayoutMode::Fill {
return PacketLookup::Filling;
}
let out = match self.buffer.pop_front() {
Some(Some(pkt)) => {
let rtp = RtpPacket::new(&pkt.packet)
.expect("FATAL: earlier valid packet now invalid (fetch)");
// The curr_ts captures the current playout point; we want to
// be able to emit *all* packets with a smaller timestamp.
// However, we need to handle this in a wrap-safe way.
// ts_diff shows where the current time lies if we treat packet_ts
// as 0, s.t. ts_diff >= 0 (equiv) packet_time <= curr_time.
let curr_ts = self.current_timestamp.as_mut().unwrap();
let pkt_ts = rtp.get_timestamp().0;
let ts_diff = (*curr_ts - pkt_ts).0 as i32;
// At least one client in the wild has seen unusual timestamp behaviour: the
// first packet sent out in a run of audio may have an older timestamp.
// This could be badly timestamped, or could conceivably be an orphaned packet
// from a prior run, or e.g.:
// (n x RTP) -> [>100ms delay] -> (RTP) -> [long O(s) delay] -> (m x RTP)
// This leaves us with two adjacent packets in the same playout with wildly varying
// timestamps. We have a slightly tricky situation -- we need to preserve accurate
// timing to correctly drain/refill/recreate very small pauses in audio, but don't
// want to block indefinitely.
//
// We have a compromise here -- if an adjacent (Drain) packet has a ts gap
// larger than it would take to go through multiple Fill/Drain cycles, then
// treat its TS as the next expected value to avoid jamming the buffer and losing
// later audio.
let skip_after = i32::try_from(
usize::from(config.playout_buffer_length.get()) * 5 * MONO_FRAME_SIZE,
)
.unwrap_or((AUDIO_FRAME_RATE * 2 * MONO_FRAME_SIZE) as i32);
if ts_diff >= 0 {
// At or before expected timestamp.
self.next_seq = (rtp.get_sequence() + 1).0;
PacketLookup::Packet(pkt)
} else if ts_diff <= -skip_after {
// >5 playouts ahead.
self.next_seq = (rtp.get_sequence() + 1).0;
*curr_ts = pkt_ts;
PacketLookup::Packet(pkt)
} else {
trace!("Witholding packet: ts_diff is {ts_diff}");
self.buffer.push_front(Some(pkt));
self.playout_mode = PlayoutMode::Fill;
PacketLookup::Filling
}
},
Some(None) => {
self.next_seq += 1;
PacketLookup::MissedPacket
},
None => PacketLookup::Filling,
};
if self.buffer.is_empty() {
self.playout_mode = PlayoutMode::Fill;
self.current_timestamp = None;
}
if let Some(ts) = self.current_timestamp.as_mut() {
*ts += &(MONO_FRAME_SIZE as u32);
}
out
}
pub fn next_seq(&self) -> RtpSequence {
self.next_seq
}
}
#[inline]
fn reset_timeout(packet: &RtpPacket<'_>, config: &Config) -> RtpTimestamp {
let t_shift = MONO_FRAME_SIZE * usize::from(config.playout_buffer_length.get());
(packet.get_timestamp() + (t_shift as u32)).0
}

View File

@@ -0,0 +1,193 @@
use super::*;
use crate::{
driver::{
tasks::error::{Error, Result},
Channels,
DecodeConfig,
DecodeMode,
},
events::context_data::{RtpData, VoiceData},
};
use discortp::{rtp::RtpExtensionPacket, Packet, PacketSize};
use opus2::{Decoder as OpusDecoder, ErrorCode};
use tracing::{error, warn};
#[derive(Debug)]
pub struct SsrcState {
playout_buffer: PlayoutBuffer,
crypto_mode: CryptoMode,
decoder: OpusDecoder,
decode_size: PacketDecodeSize,
pub(crate) prune_time: Instant,
pub(crate) disconnected: bool,
channels: Channels,
}
impl SsrcState {
pub fn new(pkt: &RtpPacket<'_>, crypto_mode: CryptoMode, config: &Config) -> Self {
let playout_capacity = config.playout_buffer_length.get() + config.playout_spike_length;
let (sample_rate, channels) = match config.decode_mode {
DecodeMode::Decode(config) => (config.sample_rate, config.channels),
DecodeMode::Decrypt | DecodeMode::Pass => Default::default(),
};
Self {
playout_buffer: PlayoutBuffer::new(usize::from(playout_capacity), pkt.get_sequence().0),
crypto_mode,
decoder: OpusDecoder::new(sample_rate.into(), channels.into())
.expect("Failed to create new Opus decoder for source."),
decode_size: PacketDecodeSize::TwentyMillis,
prune_time: Instant::now() + config.decode_state_timeout.into(),
disconnected: false,
channels,
}
}
pub fn reconfigure_decoder(&mut self, config: DecodeConfig) {
self.decoder = OpusDecoder::new(config.sample_rate.into(), config.channels.into())
.expect("Failed to create new Opus decoder for source.");
self.channels = config.channels;
}
pub fn store_packet(&mut self, packet: StoredPacket, config: &Config) {
self.playout_buffer.store_packet(packet, config);
}
pub fn refresh_timer(&mut self, state_timeout: Duration) {
if !self.disconnected {
self.prune_time = Instant::now() + state_timeout;
}
}
pub fn get_voice_tick(&mut self, config: &Config) -> Result<Option<VoiceData>> {
// Acquire a packet from the playout buffer:
// Update nexts, lasts...
// different cases: null packet who we want to decode as a miss, and packet who we must ignore temporarily.
let m_pkt = self.playout_buffer.fetch_packet(config);
let pkt = match m_pkt {
PacketLookup::Packet(StoredPacket { packet, decrypted }) => Some((packet, decrypted)),
PacketLookup::MissedPacket => None,
PacketLookup::Filling => return Ok(None),
};
let mut out = VoiceData {
packet: None,
decoded_voice: None,
};
let should_decode = config.decode_mode.should_decode();
if let Some((packet, decrypted)) = pkt {
let rtp = RtpPacket::new(&packet).unwrap();
let extensions = rtp.get_extension() != 0;
let payload = rtp.payload();
let payload_offset = self.crypto_mode.payload_prefix_len();
let payload_end_pad = payload.len() - self.crypto_mode.payload_suffix_len();
// We still need to compute missed packets here in case of long loss chains or similar.
// This occurs due to the fallback in 'store_packet' (i.e., empty buffer and massive seq difference).
// Normal losses should be handled by the below `else` branch.
let new_seq: u16 = rtp.get_sequence().into();
let missed_packets = new_seq.saturating_sub(self.playout_buffer.next_seq().0);
// TODO: maybe hand over audio and extension indices alongside packet?
let (audio, _packet_size) = self.scan_and_decode(
&payload[payload_offset..payload_end_pad],
extensions,
missed_packets,
should_decode && decrypted,
)?;
let rtp_data = RtpData {
packet,
payload_offset,
payload_end_pad,
};
out.packet = Some(rtp_data);
out.decoded_voice = audio;
} else if should_decode {
let mut audio = vec![0; self.decode_size.len()];
let dest_samples = (&mut audio[..])
.try_into()
.expect("Decode logic will cap decode buffer size at i32::MAX.");
let len = self.decoder.decode(&[], dest_samples, false)?;
audio.truncate(2 * len);
out.decoded_voice = Some(audio);
}
Ok(Some(out))
}
fn scan_and_decode(
&mut self,
data: &[u8],
extension: bool,
missed_packets: u16,
decode: bool,
) -> Result<(Option<Vec<i16>>, usize)> {
let start = if extension {
RtpExtensionPacket::new(data)
.map(|pkt| pkt.packet_size())
.ok_or_else(|| {
error!("Extension packet indicated, but insufficient space.");
Error::IllegalVoicePacket
})
} else {
Ok(0)
}?;
let pkt = if decode {
let mut out = vec![0; self.decode_size.len()];
for _ in 0..missed_packets {
let dest_samples = (&mut out[..])
.try_into()
.expect("Decode logic will cap decode buffer size at i32::MAX.");
if let Err(e) = self.decoder.decode(&[], dest_samples, false) {
warn!("Issue while decoding for missed packet: {:?}.", e);
}
}
// In general, we should expect 20 ms frames.
// However, Discord occasionally like to surprise us with something bigger.
// This is *sender-dependent behaviour*.
//
// This should scan up to find the "correct" size that a source is using,
// and then remember that.
loop {
let tried_audio_len = self.decoder.decode(&data[start..], &mut out, false);
match tried_audio_len {
Ok(audio_len) => {
// Decoding to stereo: audio_len refers to sample count irrespective of channel count.
// => multiply by number of channels.
out.truncate(self.channels.channels() * audio_len);
break;
},
Err(e) if e.code() == ErrorCode::BufferTooSmall => {
if self.decode_size.can_bump_up() {
self.decode_size = self.decode_size.bump_up();
out = vec![0; self.decode_size.len()];
} else {
error!("Received packet larger than Opus standard maximum,");
return Err(Error::IllegalVoicePacket);
}
},
Err(e) => {
error!("Failed to decode received packet: {:?}.", e);
return Err(e.into());
},
}
}
Some(out)
} else {
None
};
Ok((pkt, data.len() - start))
}
}

View File

@@ -0,0 +1,559 @@
use super::message::*;
use crate::{
driver::tasks::error::DaveReinitError,
events::CoreContext,
model::{
payload::{Heartbeat, Speaking},
CloseCode as VoiceCloseCode,
Event as GatewayEvent,
FromPrimitive,
SpeakingState,
},
ws::{Error as WsError, WsStream},
ConnectionInfo,
};
use flume::Receiver;
use rand::{distr::Uniform, Rng};
use serenity_voice_model::{
id::UserId,
payload::{
DaveMlsCommitWelcome,
DaveMlsInvalidCommitWelcome,
DaveMlsKeyPackage,
DaveMlsProposalsOperationType,
DaveTransitionReady,
},
};
use std::{
collections::{HashMap, HashSet},
num::NonZeroU16,
sync::{
atomic::{AtomicU16, Ordering},
Arc,
RwLock,
},
time::Duration,
};
use tokio::{
select,
time::{sleep_until, Instant},
};
#[cfg(feature = "tungstenite")]
use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
use tracing::{debug, info, instrument, trace, warn};
pub(crate) struct AuxNetwork {
rx: Receiver<WsMessage>,
ws_client: WsStream,
dont_send: bool,
ssrc: u32,
heartbeat_interval: Duration,
speaking: SpeakingState,
last_heartbeat_nonce: Option<u64>,
attempt_idx: usize,
info: ConnectionInfo,
dave_session: Arc<RwLock<Option<davey::DaveSession>>>,
dave_protocol_version: Arc<AtomicU16>,
dave_pending_transitions: HashMap<u16, u16>,
recognized_user_ids: HashSet<UserId>,
#[cfg(feature = "receive")]
ssrc_signalling: Arc<SsrcTracker>,
}
impl AuxNetwork {
pub(crate) fn new(
evt_rx: Receiver<WsMessage>,
ws_client: WsStream,
ssrc: u32,
heartbeat_interval: f64,
attempt_idx: usize,
info: ConnectionInfo,
dave_session: Arc<RwLock<Option<davey::DaveSession>>>,
dave_protocol_version: Arc<AtomicU16>,
#[cfg(feature = "receive")] ssrc_signalling: Arc<SsrcTracker>,
) -> Self {
let mut recognized_user_ids = HashSet::new();
recognized_user_ids.insert(info.user_id.into());
Self {
rx: evt_rx,
ws_client,
dont_send: false,
ssrc,
heartbeat_interval: Duration::from_secs_f64(heartbeat_interval / 1000.0),
speaking: SpeakingState::empty(),
last_heartbeat_nonce: None,
attempt_idx,
info,
dave_session,
dave_protocol_version,
dave_pending_transitions: HashMap::new(),
recognized_user_ids,
#[cfg(feature = "receive")]
ssrc_signalling,
}
}
#[instrument(skip(self))]
async fn run(&mut self, interconnect: &mut Interconnect) {
let mut next_heartbeat = Instant::now() + self.heartbeat_interval;
loop {
let mut ws_error = false;
let mut should_reconnect = false;
let mut ws_reason = None;
let hb = sleep_until(next_heartbeat);
select! {
// Biased polling (polling from top to bottom) is needed to process WebSocket events
// queued in the initial handshake (messages before SessionDescription). One of the
// events queued is ClientsConnect, which is needed to correctly keep track of
// recognized_user_ids and to correctly process DaveMlsProposals.
biased;
() = hb => {
ws_error = match self.send_heartbeat().await {
Err(e) => {
should_reconnect = ws_error_is_not_final(&e);
ws_reason = Some((&e).into());
true
},
_ => false,
};
next_heartbeat = self.next_heartbeat();
}
inner_msg = self.rx.recv_async() => {
match inner_msg {
Ok(WsMessage::Ws(data)) => {
self.ws_client = *data;
next_heartbeat = self.next_heartbeat();
self.dont_send = false;
},
Ok(WsMessage::ReplaceInterconnect(i)) => {
*interconnect = i;
},
Ok(WsMessage::SetKeepalive(keepalive)) => {
self.heartbeat_interval = Duration::from_secs_f64(keepalive / 1000.0);
next_heartbeat = self.next_heartbeat();
},
Ok(WsMessage::Speaking(is_speaking)) => {
if self.speaking.contains(SpeakingState::MICROPHONE) != is_speaking && !self.dont_send {
self.speaking.set(SpeakingState::MICROPHONE, is_speaking);
info!("Changing to {:?}", self.speaking);
let ssu_status = self.ws_client
.send_json(&GatewayEvent::from(Speaking {
delay: Some(0),
speaking: self.speaking,
ssrc: self.ssrc,
user_id: None,
}))
.await;
ws_error |= match ssu_status {
Err(e) => {
should_reconnect = ws_error_is_not_final(&e);
ws_reason = Some((&e).into());
true
},
_ => false,
}
}
},
Ok(WsMessage::Deliver(msg)) => {
ws_error |= match self.process_ws(interconnect, msg).await {
Err(e) => {
should_reconnect = ws_error_is_not_final(&e);
ws_reason = Some((&e).into());
true
}
_ => false,
}
},
Err(flume::RecvError::Disconnected) => {
break;
},
}
}
ws_msg = self.ws_client.recv_event_no_timeout(), if !self.dont_send => {
ws_error = match ws_msg {
Err(e) => {
should_reconnect = ws_error_is_not_final(&e);
ws_reason = Some((&e).into());
true
},
Ok(Some(msg)) => {
match self.process_ws(interconnect, msg).await {
Err(e) => {
should_reconnect = ws_error_is_not_final(&e);
ws_reason = Some((&e).into());
true
},
_ => false
}
},
_ => false,
};
}
}
if ws_error {
self.dont_send = true;
if should_reconnect {
drop(interconnect.core.send(CoreMessage::Reconnect));
} else {
drop(interconnect.core.send(CoreMessage::SignalWsClosure(
self.attempt_idx,
self.info.clone(),
ws_reason,
)));
break;
}
}
}
}
fn next_heartbeat(&self) -> Instant {
Instant::now() + self.heartbeat_interval
}
async fn send_heartbeat(&mut self) -> Result<(), WsError> {
// Discord have suddenly, mysteriously, started rejecting
// ints-as-strings. Keep JS happy here, I suppose...
const JS_MAX_INT: u64 = (1u64 << 53) - 1;
let nonce_range =
Uniform::new(0, JS_MAX_INT).expect("uniform range is finite and nonempty");
let nonce = rand::rng().sample(nonce_range);
self.last_heartbeat_nonce = Some(nonce);
trace!("Sent heartbeat {:?}", self.speaking);
if !self.dont_send {
self.ws_client
.send_json(&GatewayEvent::from(Heartbeat { nonce }))
.await?;
}
Ok(())
}
async fn process_ws(
&mut self,
interconnect: &Interconnect,
value: GatewayEvent,
) -> Result<(), WsError> {
match value {
GatewayEvent::Speaking(ev) => {
// Jarvis patch: the event goes out before the SSRC is named to the UDP task, so the
// packets that task held for it (udp_rx/held.rs) reach event handlers after it.
drop(interconnect.events.send(EventMessage::FireCoreEvent(
CoreContext::SpeakingStateUpdate(ev),
)));
#[cfg(feature = "receive")]
if let Some(user_id) = &ev.user_id {
self.ssrc_signalling.user_ssrc_map.insert(*user_id, ev.ssrc);
self.ssrc_signalling.ssrc_user_map.insert(ev.ssrc, *user_id);
}
},
GatewayEvent::ClientConnect(ev) => {
debug!("Received discontinued ClientConnect: {:?}", ev);
},
GatewayEvent::ClientDisconnect(ev) => {
#[cfg(feature = "receive")]
{
self.ssrc_signalling.disconnected_users.insert(ev.user_id);
}
self.recognized_user_ids.remove(&ev.user_id);
drop(interconnect.events.send(EventMessage::FireCoreEvent(
CoreContext::ClientDisconnect(ev),
)));
},
GatewayEvent::ClientsConnect(ev) => {
self.recognized_user_ids.extend(&ev.user_ids);
},
GatewayEvent::HeartbeatAck(ev) => {
if let Some(nonce) = self.last_heartbeat_nonce.take() {
if ev.nonce == nonce {
trace!("Heartbeat ACK received.");
} else {
warn!(
"Heartbeat nonce mismatch! Expected {}, saw {}.",
nonce, ev.nonce
);
}
}
},
GatewayEvent::DavePrepareTransition(ev) => {
self.dave_pending_transitions
.insert(ev.transition_id, ev.protocol_version);
if ev.transition_id == 0 {
self.execute_dave_transition(ev.transition_id);
} else if ev.protocol_version == 0 {
if let Some(ref mut dave_session) = *self.dave_session.write().unwrap() {
dave_session.set_passthrough_mode(true, Some(120));
}
self.ws_client
.send_json(&GatewayEvent::from(DaveTransitionReady {
transition_id: ev.transition_id,
protocol_version: ev.protocol_version,
}))
.await?;
}
},
GatewayEvent::DaveExecuteTransition(ev) => {
self.execute_dave_transition(ev.transition_id);
},
GatewayEvent::DavePrepareEpoch(ev) if ev.epoch == 1 => {
self.dave_protocol_version
.store(ev.protocol_version, Ordering::Relaxed);
match self.reinit_dave_session().await {
Err(DaveReinitError::Ws(e)) => return Err(e),
Err(e) => {
warn!(error = ?e, "failed to reinitialize DAVE session");
},
_ => {},
}
},
GatewayEvent::DaveMlsExternalSender(ev) => {
if let Some(ref mut dave_session) = *self.dave_session.write().unwrap() {
if let Err(e) = dave_session.set_external_sender(&ev.external_sender) {
warn!(error = ?e, "error setting MLS external sender");
}
}
},
GatewayEvent::DaveMlsProposals(ev) => {
let operation_type = match ev.operation_type {
DaveMlsProposalsOperationType::Append => davey::ProposalsOperationType::APPEND,
DaveMlsProposalsOperationType::Revoke => davey::ProposalsOperationType::REVOKE,
};
let result = if let Some(ref mut dave_session) = *self.dave_session.write().unwrap()
{
match dave_session.process_proposals(
operation_type,
&ev.proposals,
Some(
&self
.recognized_user_ids
.iter()
.map(|u| u.0)
.collect::<Vec<_>>(),
),
) {
Ok(result) => result,
Err(e) => {
warn!(error = ?e, "error processing MLS proposals");
None
},
}
} else {
None
};
if let Some(commit_welcome) = result {
self.ws_client
.send_binary(&GatewayEvent::from(DaveMlsCommitWelcome {
commit: commit_welcome.commit,
welcome: commit_welcome.welcome,
}))
.await?;
}
},
GatewayEvent::DaveMlsAnnounceCommitTransition(ev) => {
match self.dave_process_commit(&ev.commit_message) {
Some(Ok(())) if ev.transition_id != 0 => {
let protocol_version = self.dave_protocol_version.load(Ordering::Relaxed);
self.dave_pending_transitions
.insert(ev.transition_id, protocol_version);
self.ws_client
.send_json(&GatewayEvent::from(DaveTransitionReady {
transition_id: ev.transition_id,
protocol_version,
}))
.await?;
},
Some(Err(e)) => {
warn!("MLS commit errored: {e:?}");
self.ws_client
.send_json(&GatewayEvent::from(DaveMlsInvalidCommitWelcome {
transition_id: ev.transition_id,
}))
.await?;
match self.reinit_dave_session().await {
Err(DaveReinitError::Ws(e)) => return Err(e),
Err(e) => {
warn!(error = ?e, "failed to reinitialize DAVE session");
},
_ => {},
}
},
Some(Ok(())) | None => {},
}
},
GatewayEvent::DaveMlsWelcome(ev) => match self.dave_process_welcome(&ev.welcome) {
Some(Ok(())) if ev.transition_id != 0 => {
let protocol_version = self.dave_protocol_version.load(Ordering::Relaxed);
self.dave_pending_transitions
.insert(ev.transition_id, protocol_version);
self.ws_client
.send_json(&GatewayEvent::from(DaveTransitionReady {
transition_id: ev.transition_id,
protocol_version,
}))
.await?;
},
Some(Err(e)) => {
warn!("MLS welcome errored: {e:?}");
self.ws_client
.send_json(&GatewayEvent::from(DaveMlsInvalidCommitWelcome {
transition_id: ev.transition_id,
}))
.await?;
match self.reinit_dave_session().await {
Err(DaveReinitError::Ws(e)) => return Err(e),
Err(e) => {
warn!(error = ?e, "failed to reinitialize DAVE session");
},
_ => {},
}
},
Some(Ok(())) | None => {},
},
other => {
trace!("Received other websocket data: {:?}", other);
},
}
Ok(())
}
fn dave_process_commit(
&mut self,
commit_message: &[u8],
) -> Option<Result<(), davey::errors::ProcessCommitError>> {
let mut dave_session_lock = self.dave_session.write().unwrap();
let dave_session = (*dave_session_lock).as_mut()?;
Some(dave_session.process_commit(commit_message))
}
fn dave_process_welcome(
&mut self,
welcome: &[u8],
) -> Option<Result<(), davey::errors::ProcessWelcomeError>> {
let mut dave_session_lock = self.dave_session.write().unwrap();
let dave_session = (*dave_session_lock).as_mut()?;
Some(dave_session.process_welcome(welcome))
}
async fn reinit_dave_session(&mut self) -> Result<(), DaveReinitError> {
let protocol_version = self.dave_protocol_version.load(Ordering::Relaxed);
if let Some(dave_protocol_version) = NonZeroU16::new(protocol_version) {
let user_id = self.info.user_id.0.into();
let channel_id = self.info.channel_id.0.into();
let key_package =
if let Some(ref mut dave_session) = *self.dave_session.write().unwrap() {
dave_session.reinit(dave_protocol_version, user_id, channel_id, None)?;
dave_session.create_key_package()?
} else {
let mut dave_session =
davey::DaveSession::new(dave_protocol_version, user_id, channel_id, None)?;
let key_package = dave_session.create_key_package()?;
*self.dave_session.write().unwrap() = Some(dave_session);
key_package
};
self.ws_client
.send_binary(&GatewayEvent::DaveMlsKeyPackage(DaveMlsKeyPackage {
key_package,
}))
.await?;
} else if let Some(ref mut dave_session) = *self.dave_session.write().unwrap() {
dave_session.reset()?;
dave_session.set_passthrough_mode(true, Some(10));
}
Ok(())
}
fn execute_dave_transition(&mut self, transition_id: u16) {
let Some(new_version) = self.dave_pending_transitions.get(&transition_id).copied() else {
warn!("Received DaveExecuteTransition for unknown transition ID {transition_id}");
return;
};
let old_version = self.dave_protocol_version.load(Ordering::Relaxed);
self.dave_protocol_version
.store(new_version, Ordering::Relaxed);
// Upgraded from transport-only encryption
if transition_id > 0 && old_version == 0 && new_version != 0 {
if let Some(ref mut dave_session) = *self.dave_session.write().unwrap() {
dave_session.set_passthrough_mode(true, Some(10));
}
}
self.dave_pending_transitions.remove(&transition_id);
}
}
#[instrument(skip(interconnect, aux))]
pub(crate) async fn runner(mut interconnect: Interconnect, mut aux: AuxNetwork) {
trace!("WS thread started.");
aux.run(&mut interconnect).await;
trace!("WS thread finished.");
}
fn ws_error_is_not_final(err: &WsError) -> bool {
match err {
#[cfg(feature = "tungstenite")]
WsError::WsClosed(Some(frame)) => match frame.code {
CloseCode::Library(l) =>
if let Some(code) = VoiceCloseCode::from_u16(l) {
code.should_resume()
} else {
true
},
_ => true,
},
#[cfg(feature = "tws")]
WsError::WsClosed(Some(code)) => match (*code).into() {
code @ 4000..=4999_u16 =>
if let Some(code) = VoiceCloseCode::from_u16(code) {
code.should_resume()
} else {
true
},
_ => true,
},
e => {
debug!("Error sending/receiving ws {:?}.", e);
true
},
}
}

View File

@@ -0,0 +1,267 @@
#![allow(missing_docs)]
use flume::{Receiver, Sender};
use crate::{
tracks::{PlayMode, TrackHandle, TrackState},
Event,
EventContext,
EventHandler,
TrackEvent,
};
use std::time::Duration;
#[allow(dead_code)]
#[derive(Clone, Debug)]
pub enum TickStyle {
Timed,
UntimedWithExecLimit(Receiver<u64>),
}
#[derive(Clone, Debug, PartialEq)]
pub enum OutputMessage {
Passthrough(Vec<u8>),
Mixed(Vec<f32>),
Silent,
}
#[allow(dead_code)]
impl OutputMessage {
#[must_use]
pub fn is_passthrough(&self) -> bool {
matches!(self, Self::Passthrough(_))
}
#[must_use]
pub fn is_mixed(&self) -> bool {
matches!(self, Self::Mixed(_))
}
#[must_use]
pub fn is_mixed_with_nonzero_signal(&self) -> bool {
if let Self::Mixed(data) = self {
data.iter().any(|v| *v != 0.0f32)
} else {
false
}
}
#[must_use]
pub fn is_explicit_silence(&self) -> bool {
*self == Self::Silent
}
}
#[allow(dead_code)]
#[derive(Clone, Debug)]
pub enum OutputMode {
Raw(Sender<TickMessage<OutputMessage>>),
Rtp(Sender<TickMessage<Vec<u8>>>),
}
#[allow(dead_code)]
#[derive(Clone, Debug)]
pub enum TickMessage<T> {
El(T),
NoEl,
}
impl<T> From<T> for TickMessage<T> {
fn from(val: T) -> Self {
TickMessage::El(val)
}
}
impl From<TickMessage<OutputMessage>> for OutputPacket {
fn from(val: TickMessage<OutputMessage>) -> Self {
match val {
TickMessage::El(e) => OutputPacket::Raw(e),
TickMessage::NoEl => OutputPacket::Empty,
}
}
}
impl From<TickMessage<Vec<u8>>> for OutputPacket {
fn from(val: TickMessage<Vec<u8>>) -> Self {
match val {
TickMessage::El(e) => OutputPacket::Rtp(e),
TickMessage::NoEl => OutputPacket::Empty,
}
}
}
#[derive(Clone, Debug, PartialEq)]
pub enum OutputPacket {
Raw(OutputMessage),
Rtp(Vec<u8>),
Empty,
}
impl OutputPacket {
#[must_use]
pub fn raw(&self) -> Option<&OutputMessage> {
if let Self::Raw(o) = self {
Some(o)
} else {
None
}
}
}
#[derive(Clone, Debug)]
pub enum OutputReceiver {
Raw(Receiver<TickMessage<OutputMessage>>),
Rtp(Receiver<TickMessage<Vec<u8>>>),
}
#[derive(Clone)]
pub struct DriverTestHandle {
pub rx: OutputReceiver,
pub tx: Sender<u64>,
}
impl DriverTestHandle {
#[must_use]
pub fn recv(&self) -> OutputPacket {
match &self.rx {
OutputReceiver::Raw(rx) => rx.recv().unwrap().into(),
OutputReceiver::Rtp(rx) => rx.recv().unwrap().into(),
}
}
pub async fn recv_async(&self) -> OutputPacket {
match &self.rx {
OutputReceiver::Raw(rx) => rx.recv_async().await.unwrap().into(),
OutputReceiver::Rtp(rx) => rx.recv_async().await.unwrap().into(),
}
}
#[must_use]
pub fn len(&self) -> usize {
match &self.rx {
OutputReceiver::Raw(rx) => rx.len(),
OutputReceiver::Rtp(rx) => rx.len(),
}
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn wait(&self, n_ticks: u64) {
for _i in 0..n_ticks {
drop(self.recv());
}
}
pub async fn wait_async(&self, n_ticks: u64) {
for _i in 0..n_ticks {
drop(self.recv_async().await);
}
}
pub fn spawn_ticker(&self) {
let remote = self.clone();
tokio::spawn(async move {
loop {
remote.skip(1).await;
tokio::time::sleep(Duration::from_millis(1)).await;
}
});
}
pub fn wait_noisy(&self, n_ticks: u64) {
for _i in 0..n_ticks {
match self.recv() {
OutputPacket::Empty => eprintln!("pkt: Nothing"),
OutputPacket::Rtp(p) => eprintln!("pkt: RTP[{}B]", p.len()),
OutputPacket::Raw(OutputMessage::Silent) => eprintln!("pkt: Raw-Silent"),
OutputPacket::Raw(OutputMessage::Passthrough(p)) => {
eprintln!("pkt: Raw-Passthrough[{}B]", p.len());
},
OutputPacket::Raw(OutputMessage::Mixed(p)) => {
eprintln!("pkt: Raw-Mixed[{}B]", p.len());
},
}
}
}
pub async fn skip(&self, n_ticks: u64) {
self.tick(n_ticks);
self.wait_async(n_ticks).await;
}
pub fn tick(&self, n_ticks: u64) {
assert!(
n_ticks != 0,
"Number of ticks to advance driver/mixer must be >= 1."
);
self.tx.send(n_ticks).unwrap();
}
pub async fn ready_track(
&self,
handle: &TrackHandle,
tick_wait: Option<Duration>,
) -> TrackState {
struct SongPlayable {
tx: Sender<TrackState>,
}
#[async_trait::async_trait]
impl EventHandler for SongPlayable {
async fn act(&self, ctx: &crate::EventContext<'_>) -> Option<Event> {
if let EventContext::Track(&[(state, _)]) = ctx {
drop(self.tx.send(state.clone()));
}
Some(Event::Cancel)
}
}
struct SongErred {
tx: Sender<PlayMode>,
}
#[async_trait::async_trait]
impl EventHandler for SongErred {
async fn act(&self, ctx: &crate::EventContext<'_>) -> Option<Event> {
if let EventContext::Track(&[(state, _)]) = ctx {
drop(self.tx.send(state.playing.clone()));
}
Some(Event::Cancel)
}
}
let (tx, rx) = flume::bounded(1);
let (err_tx, err_rx) = flume::bounded(1);
handle
.add_event(Event::Track(TrackEvent::Playable), SongPlayable { tx })
.expect("Adding track evt should not fail before any ticks.");
handle
.add_event(Event::Track(TrackEvent::Error), SongErred { tx: err_tx })
.expect("Adding track evt should not fail before any ticks.");
loop {
self.tick(1);
tokio::time::sleep(tick_wait.unwrap_or_else(|| Duration::from_millis(20))).await;
self.wait_async(1).await;
match err_rx.try_recv() {
Ok(e) => panic!("Error reported on track: {e:?}"),
Err(flume::TryRecvError::Empty | flume::TryRecvError::Disconnected) => {},
}
match rx.try_recv() {
Ok(val) => return val,
Err(flume::TryRecvError::Disconnected) => panic!(),
Err(flume::TryRecvError::Empty) => {},
}
}
}
}

View File

@@ -0,0 +1,255 @@
#![allow(missing_docs)]
use super::{
scheduler::*,
tasks::{message::*, mixer::Mixer},
*,
};
use crate::{
constants::*,
input::{
cached::Compressed,
codecs::{get_codec_registry, get_probe},
RawAdapter,
},
test_utils,
tracks::LoopState,
};
use flume::Receiver;
use std::{
io::Cursor,
net::UdpSocket,
num::NonZeroU16,
sync::{atomic::AtomicU16, Arc, RwLock},
};
use tokio::runtime::Handle;
// create a dummied task + interconnect.
// measure perf at varying numbers of sources (binary 1--64) without passthrough support.
#[cfg(feature = "receive")]
pub type Listeners = (
Receiver<CoreMessage>,
Receiver<EventMessage>,
Receiver<UdpRxMessage>,
);
#[cfg(not(feature = "receive"))]
pub type Listeners = (Receiver<CoreMessage>, Receiver<EventMessage>);
pub type DummyMixer = (Mixer, Listeners);
impl Mixer {
#[must_use]
pub fn mock(handle: Handle, softclip: bool) -> DummyMixer {
let (mix_tx, mix_rx) = flume::unbounded();
let (core_tx, core_rx) = flume::unbounded();
let (event_tx, event_rx) = flume::unbounded();
#[cfg(feature = "receive")]
let (udp_receiver_tx, udp_receiver_rx) = flume::unbounded();
let ic = Interconnect {
core: core_tx,
events: event_tx,
mixer: mix_tx,
};
// Scheduler must be created from a Tokio context...
let (tx, rx) = flume::unbounded();
handle.spawn_blocking(move || tx.send(crate::Config::default().use_softclip(softclip)));
let config = rx.recv().unwrap();
let mut out = Mixer::new(mix_rx, handle, ic, config);
let udp_tx = UdpSocket::bind("0.0.0.0:0").expect("Failed to create send port.");
udp_tx
.connect("127.0.0.1:5316")
.expect("Failed to connect to local dest port.");
let mode = CryptoMode::Aes256Gcm;
let cipher = mode.cipher_from_key(&[0u8; 32]).unwrap();
let crypto_state = mode.into();
let dave_protocol_version = Arc::new(AtomicU16::new(davey::DAVE_PROTOCOL_VERSION));
let dave_session = Arc::new(RwLock::new(Some(
davey::DaveSession::new(
NonZeroU16::new(1).expect("failed to initialize NonZeroU16 from static value"),
1,
1,
None,
)
.expect("failed to initialize davey::DaveSession"),
)));
#[cfg(feature = "receive")]
let fake_conn = MixerConnection {
cipher,
crypto_state,
dave_session,
dave_protocol_version,
udp_rx: udp_receiver_tx,
udp_tx,
};
#[cfg(not(feature = "receive"))]
let fake_conn = MixerConnection {
cipher,
crypto_state,
dave_session,
dave_protocol_version,
udp_tx,
};
out.conn_active = Some(fake_conn);
#[cfg(feature = "receive")]
return (out, (core_rx, event_rx, udp_receiver_rx));
#[cfg(not(feature = "receive"))]
return (out, (core_rx, event_rx));
}
#[must_use]
pub fn test_with_float(num_tracks: usize, handle: Handle, softclip: bool) -> DummyMixer {
let mut out = Self::mock(handle, softclip);
let floats = test_utils::make_sine(10 * STEREO_FRAME_SIZE, true);
for _ in 0..num_tracks {
let input: Input = RawAdapter::new(Cursor::new(floats.clone()), 48_000, 2).into();
let promoted = match input {
Input::Live(l, _) => l.promote(get_codec_registry(), get_probe()),
Input::Lazy(_) => panic!("Failed to create a guaranteed source."),
};
let (_, ctx) = Track::from(Input::Live(promoted.unwrap(), None)).into_context();
_ = out.0.add_track(ctx);
}
out
}
#[must_use]
pub fn test_with_float_unending(handle: Handle, softclip: bool) -> (DummyMixer, TrackHandle) {
let mut out = Self::mock(handle, softclip);
let floats = test_utils::make_sine(10 * STEREO_FRAME_SIZE, true);
let input: Input = RawAdapter::new(Cursor::new(floats.clone()), 48_000, 2).into();
let promoted = match input {
Input::Live(l, _) => l.promote(get_codec_registry(), get_probe()),
Input::Lazy(_) => panic!("Failed to create a guaranteed source."),
};
let mut track = Track::from(Input::Live(promoted.unwrap(), None));
track.loops = LoopState::Infinite;
let (handle, ctx) = track.into_context();
_ = out.0.add_track(ctx);
(out, handle)
}
#[must_use]
pub fn test_with_float_drop(num_tracks: usize, handle: Handle) -> DummyMixer {
let mut out = Self::mock(handle, true);
for i in 0..num_tracks {
let floats = test_utils::make_sine((i / 5) * STEREO_FRAME_SIZE, true);
let input: Input = RawAdapter::new(Cursor::new(floats.clone()), 48_000, 2).into();
let promoted = match input {
Input::Live(l, _) => l.promote(get_codec_registry(), get_probe()),
Input::Lazy(_) => panic!("Failed to create a guaranteed source."),
};
let (_, ctx) = Track::from(Input::Live(promoted.unwrap(), None)).into_context();
_ = out.0.add_track(ctx);
}
out
}
#[must_use]
pub fn test_with_opus(handle: &Handle) -> DummyMixer {
// should add a single opus-based track.
// make this fully loaded to prevent any perf cost there.
let mut out = Self::mock(handle.clone(), false);
let floats = test_utils::make_sine(6 * STEREO_FRAME_SIZE, true);
let input: Input = RawAdapter::new(Cursor::new(floats), 48_000, 2).into();
let mut src = handle.block_on(async move {
Compressed::new(input, Bitrate::Bits(128_000))
.await
.expect("These parameters are well-defined.")
});
src.raw.load_all();
let promoted = match src.into() {
Input::Live(l, _) => l.promote(get_codec_registry(), get_probe()),
Input::Lazy(_) => panic!("Failed to create a guaranteed source."),
};
let (_, ctx) = Track::from(Input::Live(promoted.unwrap(), None)).into_context();
_ = out.0.add_track(ctx);
out
}
}
pub struct MockScheduler {
pub core: Live,
pub stats: Arc<StatBlock>,
pub local: Arc<LiveStatBlock>,
pub rx: Receiver<SchedulerMessage>,
pub tx: Sender<(TaskId, ParkedMixer)>,
pub id: TaskId,
}
impl MockScheduler {
#[must_use]
pub fn new(mode: Option<Mode>) -> Self {
let stats = Arc::new(StatBlock::default());
let local = Arc::new(LiveStatBlock::default());
let (task_tx, task_rx) = flume::unbounded();
let (sched_tx, sched_rx) = flume::unbounded();
let cfg = crate::driver::SchedulerConfig {
strategy: mode.unwrap_or_default(),
move_expensive_tasks: true,
};
let core = Live::new(
WorkerId::new(),
cfg,
stats.clone(),
local.clone(),
task_rx,
sched_tx,
);
Self {
core,
stats,
local,
rx: sched_rx,
tx: task_tx,
id: TaskId::new(),
}
}
pub fn add_mixer_direct(&mut self, m: Mixer) {
let id = self.id.incr();
self.core.add_task_direct(m, id);
}
#[must_use]
pub fn from_mixers(mode: Option<Mode>, mixers: Vec<DummyMixer>) -> (Self, Vec<Listeners>) {
let mut out = Self::new(mode);
let mut listeners = vec![];
for (mixer, listener) in mixers {
out.add_mixer_direct(mixer);
listeners.push(listener);
}
(out, listeners)
}
}

148
app/vendor/songbird/src/error.rs vendored Normal file
View File

@@ -0,0 +1,148 @@
//! Driver and gateway error handling.
#[cfg(feature = "serenity")]
use futures::channel::mpsc::TrySendError;
pub use serde_json::Error as JsonError;
#[cfg(feature = "serenity")]
use serenity::gateway::ShardRunnerMessage;
#[cfg(feature = "gateway")]
use std::{error::Error, fmt};
#[cfg(feature = "twilight")]
use twilight_gateway::error::ChannelError;
#[cfg(feature = "gateway")]
#[derive(Debug)]
#[non_exhaustive]
/// Error returned when a manager or call handler is
/// unable to send messages over Discord's gateway.
pub enum JoinError {
/// Request to join was dropped, cancelled, or replaced.
Dropped,
/// No available gateway connection was provided to send
/// voice state update messages.
NoSender,
/// Tried to leave a [`Call`] which was not found.
///
/// [`Call`]: crate::Call
NoCall,
/// Connection details were not received from Discord in the
/// time given in [the `Call`'s configuration].
///
/// This can occur if a message is lost by the Discord client
/// between restarts, or if Discord's gateway believes that
/// this bot is still in the channel it attempts to join.
///
/// *Users should `leave` the server on the gateway before
/// re-attempting connection.*
///
/// [the `Call`'s configuration]: crate::Config
TimedOut,
#[cfg(feature = "driver")]
/// The driver failed to establish a voice connection.
///
/// *Users should `leave` the server on the gateway before
/// re-attempting connection.*
Driver(ConnectionError),
#[cfg(feature = "serenity")]
/// Serenity-specific WebSocket send error.
Serenity(Box<TrySendError<ShardRunnerMessage>>),
#[cfg(feature = "twilight")]
/// Twilight-specific WebSocket send error when a message fails to send over websocket.
Twilight(ChannelError),
}
#[cfg(feature = "gateway")]
impl JoinError {
/// Indicates whether this failure may have left (or been
/// caused by) Discord's gateway state being in an
/// inconsistent state.
///
/// Failure to `leave` before rejoining may cause further
/// timeouts.
pub fn should_leave_server(&self) -> bool {
matches!(self, JoinError::TimedOut)
}
#[cfg(feature = "driver")]
/// Indicates whether this failure can be reattempted via
/// [`Driver::connect`] with retreived connection info.
///
/// Failure to `leave` before rejoining may cause further
/// timeouts.
///
/// [`Driver::connect`]: crate::driver::Driver
pub fn should_reconnect_driver(&self) -> bool {
matches!(self, JoinError::Driver(_))
}
}
#[cfg(feature = "gateway")]
impl fmt::Display for JoinError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "failed to join voice channel: ")?;
match self {
JoinError::Dropped => write!(f, "request was cancelled/dropped"),
JoinError::NoSender => write!(f, "no gateway destination"),
JoinError::NoCall => write!(f, "tried to leave a non-existent call"),
JoinError::TimedOut => write!(f, "gateway response from Discord timed out"),
#[cfg(feature = "driver")]
JoinError::Driver(_) => write!(f, "establishing connection failed"),
#[cfg(feature = "serenity")]
JoinError::Serenity(e) => e.fmt(f),
#[cfg(feature = "twilight")]
JoinError::Twilight(e) => e.fmt(f),
}
}
}
#[cfg(feature = "gateway")]
impl Error for JoinError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
JoinError::Dropped => None,
JoinError::NoSender => None,
JoinError::NoCall => None,
JoinError::TimedOut => None,
#[cfg(feature = "driver")]
JoinError::Driver(e) => Some(e),
#[cfg(feature = "serenity")]
JoinError::Serenity(e) => e.source(),
#[cfg(feature = "twilight")]
JoinError::Twilight(e) => e.source(),
}
}
}
#[cfg(all(feature = "serenity", feature = "gateway"))]
impl From<Box<TrySendError<ShardRunnerMessage>>> for JoinError {
fn from(e: Box<TrySendError<ShardRunnerMessage>>) -> Self {
JoinError::Serenity(e)
}
}
#[cfg(all(feature = "twilight", feature = "gateway"))]
impl From<ChannelError> for JoinError {
fn from(e: ChannelError) -> Self {
JoinError::Twilight(e)
}
}
#[cfg(all(feature = "driver", feature = "gateway"))]
impl From<ConnectionError> for JoinError {
fn from(e: ConnectionError) -> Self {
JoinError::Driver(e)
}
}
#[cfg(feature = "gateway")]
/// Convenience type for Discord gateway error handling.
pub type JoinResult<T> = Result<T, JoinError>;
#[cfg(feature = "driver")]
pub use crate::{
driver::{
connection::error::{Error as ConnectionError, Result as ConnectionResult},
SchedulerError,
},
tracks::{ControlError, PlayError, TrackResult},
};

View File

@@ -0,0 +1,31 @@
use crate::id::*;
/// Voice connection details gathered at setup/reinstantiation.
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
#[non_exhaustive]
pub struct ConnectData<'a> {
/// ID of the voice channel being joined.
///
/// This can be used to reconnect/renew a voice session via the gateway.
pub channel_id: ChannelId,
/// ID of the target voice channel's parent guild.
pub guild_id: GuildId,
/// Unique string describing this session for validation/authentication purposes.
pub session_id: &'a str,
/// The domain name of Discord's voice/TURN server.
///
/// With the introduction of Discord's automatic voice server selection,
/// this is no longer guaranteed to match a server's settings. This field
/// may be useful if you need/wish to move your voice connection to a node/shard
/// closer to Discord.
pub server: &'a str,
/// The [RTP SSRC] *("Synchronisation source")* assigned by the voice server
/// for the duration of this call.
///
/// All packets sent will use this SSRC, which is not related to the sender's User
/// ID. These are usually allocated sequentially by Discord, following on from
/// a random starting SSRC.
///
/// [RTP SSRC]: https://tools.ietf.org/html/rfc3550#section-3
pub ssrc: u32,
}

View File

@@ -0,0 +1,126 @@
use crate::{
error::ConnectionError,
id::*,
model::{CloseCode as VoiceCloseCode, FromPrimitive},
ws::Error as WsError,
};
#[cfg(feature = "tungstenite")]
use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
/// Voice connection details gathered at termination or failure.
///
/// In the event of a failure, this event data is gathered after
/// a reconnection strategy has exhausted all of its attempts.
#[derive(Debug)]
#[non_exhaustive]
pub struct DisconnectData<'a> {
/// The location that a voice connection was terminated.
pub kind: DisconnectKind,
/// The cause of any connection failure.
///
/// If `None`, then this disconnect was requested by the user in some way
/// (i.e., leaving or changing voice channels).
pub reason: Option<DisconnectReason>,
/// ID of the voice channel being joined.
///
/// This can be used to reconnect/renew a voice session via the gateway.
pub channel_id: ChannelId,
/// ID of the target voice channel's parent guild.
pub guild_id: GuildId,
/// Unique string describing this session for validation/authentication purposes.
pub session_id: &'a str,
}
/// The location that a voice connection was terminated.
#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)]
#[non_exhaustive]
pub enum DisconnectKind {
/// The voice driver failed to connect to the server.
///
/// This requires explicit handling at the gateway level
/// to either reconnect or fully disconnect.
Connect,
/// The voice driver failed to reconnect to the server.
///
/// This requires explicit handling at the gateway level
/// to either reconnect or fully disconnect.
Reconnect,
/// The voice connection was terminated mid-session by either
/// the user or Discord.
///
/// If `reason == None`, then this disconnection is either
/// a full disconnect or a user-requested channel change.
/// Otherwise, this is likely a session expiry (requiring user
/// handling to fully disconnect/reconnect).
Runtime,
}
/// The reason that a voice connection failed.
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum DisconnectReason {
/// This (re)connection attempt was dropped due to another request.
AttemptDiscarded,
/// Songbird had an internal error.
///
/// This should never happen; if this is ever seen, raise an issue with logs.
Internal,
/// A host-specific I/O error caused the fault; this is likely transient, and
/// should be retried some time later.
Io,
/// Songbird and Discord disagreed on the protocol used to establish a
/// voice connection.
///
/// This should never happen; if this is ever seen, raise an issue with logs.
ProtocolViolation,
/// A voice connection was not established in the specified time.
TimedOut,
/// The call was manually disconnected by a user command, e.g. [`Driver::leave`].
///
/// [`Driver::leave`]: crate::driver::Driver::leave
Requested,
/// The Websocket connection was closed by Discord.
///
/// This typically indicates that the voice session has expired,
/// and a new one needs to be requested via the gateway.
WsClosed(Option<VoiceCloseCode>),
}
impl From<&ConnectionError> for DisconnectReason {
fn from(e: &ConnectionError) -> Self {
match e {
ConnectionError::AttemptDiscarded => Self::AttemptDiscarded,
ConnectionError::CryptoInvalidLength
| ConnectionError::CryptoModeInvalid
| ConnectionError::CryptoModeUnavailable
| ConnectionError::DaveCreateKeyPackageError(_)
| ConnectionError::DaveInitializationError(_)
| ConnectionError::EndpointUrl
| ConnectionError::IllegalDiscoveryResponse
| ConnectionError::IllegalIp
| ConnectionError::Json(_) => Self::ProtocolViolation,
ConnectionError::Io(_) => Self::Io,
ConnectionError::Crypto(_) | ConnectionError::InterconnectFailure(_) => Self::Internal,
ConnectionError::Ws(ws) => ws.into(),
ConnectionError::TimedOut => Self::TimedOut,
}
}
}
impl From<&WsError> for DisconnectReason {
fn from(e: &WsError) -> Self {
Self::WsClosed(match e {
#[cfg(feature = "tungstenite")]
WsError::WsClosed(Some(frame)) => match frame.code {
CloseCode::Library(l) => VoiceCloseCode::from_u16(l),
_ => None,
},
#[cfg(feature = "tws")]
WsError::WsClosed(Some(code)) => match (*code).into() {
code @ 4000..=4999_u16 => VoiceCloseCode::from_u16(code),
_ => None,
},
_ => None,
})
}
}

View File

@@ -0,0 +1,18 @@
//! Types containing the main body of an [`EventContext`].
//!
//! [`EventContext`]: super::EventContext
mod connect;
mod disconnect;
#[cfg(feature = "receive")]
mod rtcp;
#[cfg(feature = "receive")]
mod rtp;
#[cfg(feature = "receive")]
mod voice;
#[cfg(feature = "receive")]
use bytes::Bytes;
pub use self::{connect::*, disconnect::*};
#[cfg(feature = "receive")]
pub use self::{rtcp::*, rtp::*, voice::*};

View File

@@ -0,0 +1,29 @@
use discortp::rtcp::RtcpPacket;
use super::*;
#[derive(Clone, Debug, Eq, PartialEq)]
#[non_exhaustive]
/// Telemetry/statistics packet, received from another stream
///
/// `payload_offset` contains the true payload location within the raw packet's `payload()`,
/// to allow manual decoding of `Rtcp` packet bodies.
pub struct RtcpData {
/// Raw RTCP packet data.
pub packet: Bytes,
/// Byte index into the packet body (after headers) for where the payload begins.
pub payload_offset: usize,
/// Number of bytes at the end of the packet to discard.
pub payload_end_pad: usize,
}
impl RtcpData {
/// Create a zero-copy view of the inner RTCP packet.
///
/// This allows easy access to packet header fields, taking them from the underlying
/// `Bytes` as needed while handling endianness etc.
pub fn rtcp(&'_ self) -> RtcpPacket<'_> {
RtcpPacket::new(&self.packet)
.expect("FATAL: leaked illegally small RTP packet from UDP Rx task.")
}
}

View File

@@ -0,0 +1,31 @@
use discortp::rtp::RtpPacket;
use super::*;
#[derive(Clone, Debug, Eq, PartialEq)]
#[non_exhaustive]
/// Opus audio packet, received from another stream
///
/// `payload_offset` contains the true payload location within the raw packet's `payload()`,
/// if extensions or raw packet data are required.
pub struct RtpData {
/// Raw RTP packet data.
///
/// Includes the SSRC (i.e., sender) of this packet.
pub packet: Bytes,
/// Byte index into the packet body (after headers) for where the payload begins.
pub payload_offset: usize,
/// Number of bytes at the end of the packet to discard.
pub payload_end_pad: usize,
}
impl RtpData {
/// Create a zero-copy view of the inner RTP packet.
///
/// This allows easy access to packet header fields, taking them from the underlying
/// `Bytes` as needed while handling endianness etc.
pub fn rtp(&'_ self) -> RtpPacket<'_> {
RtpPacket::new(&self.packet)
.expect("FATAL: leaked illegally small RTP packet from UDP Rx task.")
}
}

View File

@@ -0,0 +1,43 @@
use std::collections::{HashMap, HashSet};
use super::*;
#[derive(Clone, Debug, Eq, PartialEq)]
#[non_exhaustive]
/// Audio data from all users in a voice channel, fired every 20ms.
///
/// Songbird implements a jitter buffer to sycnhronise user packets, smooth out network latency, and
/// handle packet reordering by the network. Packet playout via this event is delayed by approximately
/// [`Config::playout_buffer_length`]` * 20ms` from its original arrival.
///
/// [`Config::playout_buffer_length`]: crate::Config::playout_buffer_length
pub struct VoiceTick {
/// Decoded voice data and source packets sent by each user.
pub speaking: HashMap<u32, VoiceData>,
/// Set of all SSRCs currently known in the call who aren't included in [`Self::speaking`].
pub silent: HashSet<u32>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
#[non_exhaustive]
/// Voice packet and audio data for a single user, from a single tick.
pub struct VoiceData {
/// RTP packet clocked out for this tick.
///
/// If `None`, then the packet was lost, and [`Self::decoded_voice`] may include
/// around one codec delay's worth of audio.
pub packet: Option<RtpData>,
/// PCM audio obtained from a user.
///
/// Valid audio data (`Some(audio)` where `audio.len >= 0`) typically contains 20ms of 16-bit PCM audio
/// using native endianness. This defaults to stereo audio at 48kHz, and can be configured via
/// [`DecodeConfig::sample_rate`] and [`DecodeConfig::sample_rate`] -- channels are interleaved
/// (i.e., `L, R, L, R, ...`) if stereo.
///
/// This value will be `None` if Songbird is not configured to decode audio.
///
/// [`DecodeConfig::decode_channels`]: crate::driver::DecodeConfig::channels
/// [`DecodeConfig::sample_rate`]: crate::driver::DecodeConfig::sample_rate
pub decoded_voice: Option<Vec<i16>>,
}

View File

@@ -0,0 +1,82 @@
use super::context_data::*;
use crate::ConnectionInfo;
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct InternalConnect {
pub info: ConnectionInfo,
pub ssrc: u32,
}
#[derive(Debug)]
pub struct InternalDisconnect {
pub kind: DisconnectKind,
pub reason: Option<DisconnectReason>,
pub info: ConnectionInfo,
}
impl<'a> From<&'a InternalConnect> for ConnectData<'a> {
fn from(val: &'a InternalConnect) -> Self {
Self {
channel_id: val.info.channel_id,
guild_id: val.info.guild_id,
session_id: &val.info.session_id,
server: &val.info.endpoint,
ssrc: val.ssrc,
}
}
}
impl<'a> From<&'a InternalDisconnect> for DisconnectData<'a> {
fn from(val: &'a InternalDisconnect) -> Self {
Self {
kind: val.kind,
reason: val.reason,
channel_id: val.info.channel_id,
guild_id: val.info.guild_id,
session_id: &val.info.session_id,
}
}
}
#[cfg(feature = "receive")]
mod receive {
use super::*;
use bytes::Bytes;
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct InternalRtpPacket {
pub packet: Bytes,
pub payload_offset: usize,
pub payload_end_pad: usize,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct InternalRtcpPacket {
pub packet: Bytes,
pub payload_offset: usize,
pub payload_end_pad: usize,
}
impl<'a> From<&'a InternalRtpPacket> for RtpData {
fn from(val: &'a InternalRtpPacket) -> Self {
Self {
packet: val.packet.clone(),
payload_offset: val.payload_offset,
payload_end_pad: val.payload_end_pad,
}
}
}
impl<'a> From<&'a InternalRtcpPacket> for RtcpData {
fn from(val: &'a InternalRtcpPacket) -> Self {
Self {
packet: val.packet.clone(),
payload_offset: val.payload_offset,
payload_end_pad: val.payload_end_pad,
}
}
}
}
#[cfg(feature = "receive")]
pub use receive::*;

View File

@@ -0,0 +1,114 @@
pub mod data;
pub(crate) mod internal_data;
use super::*;
use crate::{
model::payload::{ClientDisconnect, Speaking},
tracks::{TrackHandle, TrackState},
};
pub use data as context_data;
use data::*;
use internal_data::*;
/// Information about which tracks or data fired an event.
///
/// [`Track`] events may be local or global, and have no tracks
/// if fired on the global context via [`Driver::add_global_event`].
///
/// [`Track`]: crate::tracks::Track
/// [`Driver::add_global_event`]: crate::driver::Driver::add_global_event
#[derive(Debug)]
#[non_exhaustive]
pub enum EventContext<'a> {
/// Track event context, passed to events created via [`TrackHandle::add_event`],
/// [`EventStore::add_event`], or relevant global events.
///
/// [`EventStore::add_event`]: EventStore::add_event
/// [`TrackHandle::add_event`]: TrackHandle::add_event
Track(&'a [(&'a TrackState, &'a TrackHandle)]),
/// Speaking state update, typically describing how another voice
/// user is transmitting audio data. Clients must send at least one such
/// packet to allow SSRC/UserID matching.
SpeakingStateUpdate(Speaking),
#[cfg(feature = "receive")]
/// Reordered and decoded audio packets, received every 20ms.
VoiceTick(VoiceTick),
#[cfg(feature = "receive")]
/// Opus audio packet, received from another stream.
RtpPacket(RtpData),
#[cfg(feature = "receive")]
/// Telemetry/statistics packet, received from another stream.
RtcpPacket(RtcpData),
/// Fired whenever a client disconnects.
ClientDisconnect(ClientDisconnect),
/// Fires when this driver successfully connects to a voice channel.
DriverConnect(ConnectData<'a>),
/// Fires when this driver successfully reconnects after a network error.
DriverReconnect(ConnectData<'a>),
/// Fires when this driver fails to connect to, or drops from, a voice channel.
DriverDisconnect(DisconnectData<'a>),
}
#[derive(Debug)]
pub enum CoreContext {
SpeakingStateUpdate(Speaking),
#[cfg(feature = "receive")]
VoiceTick(VoiceTick),
#[cfg(feature = "receive")]
RtpPacket(InternalRtpPacket),
#[cfg(feature = "receive")]
RtcpPacket(InternalRtcpPacket),
ClientDisconnect(ClientDisconnect),
DriverConnect(InternalConnect),
DriverReconnect(InternalConnect),
DriverDisconnect(InternalDisconnect),
}
impl<'a> CoreContext {
pub(crate) fn to_user_context(&'a self) -> EventContext<'a> {
match self {
Self::SpeakingStateUpdate(evt) => EventContext::SpeakingStateUpdate(*evt),
#[cfg(feature = "receive")]
Self::VoiceTick(evt) => EventContext::VoiceTick(evt.clone()),
#[cfg(feature = "receive")]
Self::RtpPacket(evt) => EventContext::RtpPacket(RtpData::from(evt)),
#[cfg(feature = "receive")]
Self::RtcpPacket(evt) => EventContext::RtcpPacket(RtcpData::from(evt)),
Self::ClientDisconnect(evt) => EventContext::ClientDisconnect(*evt),
Self::DriverConnect(evt) => EventContext::DriverConnect(ConnectData::from(evt)),
Self::DriverReconnect(evt) => EventContext::DriverReconnect(ConnectData::from(evt)),
Self::DriverDisconnect(evt) =>
EventContext::DriverDisconnect(DisconnectData::from(evt)),
}
}
}
impl EventContext<'_> {
/// Retreive the event class for an event (i.e., when matching)
/// an event against the registered listeners.
#[must_use]
pub fn to_core_event(&self) -> Option<CoreEvent> {
match self {
Self::SpeakingStateUpdate(_) => Some(CoreEvent::SpeakingStateUpdate),
#[cfg(feature = "receive")]
Self::VoiceTick(_) => Some(CoreEvent::VoiceTick),
#[cfg(feature = "receive")]
Self::RtpPacket(_) => Some(CoreEvent::RtpPacket),
#[cfg(feature = "receive")]
Self::RtcpPacket(_) => Some(CoreEvent::RtcpPacket),
Self::ClientDisconnect(_) => Some(CoreEvent::ClientDisconnect),
Self::DriverConnect(_) => Some(CoreEvent::DriverConnect),
Self::DriverReconnect(_) => Some(CoreEvent::DriverReconnect),
Self::DriverDisconnect(_) => Some(CoreEvent::DriverDisconnect),
_ => None,
}
}
}

77
app/vendor/songbird/src/events/core.rs vendored Normal file
View File

@@ -0,0 +1,77 @@
/// Voice core events occur on receipt of
/// voice packets and telemetry.
///
/// Core events persist while the `action` in [`EventData`]
/// returns `None`.
///
/// ## Events from other users
/// Songbird can observe when a user *speaks for the first time* ([`SpeakingStateUpdate`]),
/// when a client leaves the session ([`ClientDisconnect`]).
///
/// When the `"receive"` feature is enabled, songbird can also handle voice packets
#[cfg_attr(feature = "receive", doc = "([`RtpPacket`](Self::RtpPacket)),")]
#[cfg_attr(not(feature = "receive"), doc = "(`RtpPacket`),")]
/// decode and track speaking users
#[cfg_attr(feature = "receive", doc = "([`VoiceTick`](Self::VoiceTick)),")]
#[cfg_attr(not(feature = "receive"), doc = "(`VoiceTick`),")]
/// and handle telemetry data
#[cfg_attr(feature = "receive", doc = "([`RtcpPacket`](Self::RtcpPacket)).")]
#[cfg_attr(not(feature = "receive"), doc = "(`RtcpPacket`).")]
/// The format of voice packets is described by
#[cfg_attr(
feature = "receive",
doc = "[`VoiceData`](super::context::data::VoiceData)."
)]
#[cfg_attr(not(feature = "receive"), doc = "`VoiceData`.")]
///
/// To detect when a user connects, you must correlate gateway (e.g., `VoiceStateUpdate`) events
/// from the main part of your bot.
///
/// To obtain a user's SSRC, you must use [`SpeakingStateUpdate`] events.
///
/// [`EventData`]: super::EventData
/// [`SpeakingStateUpdate`]: Self::SpeakingStateUpdate
/// [`ClientDisconnect`]: Self::ClientDisconnect
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, enum_map::Enum)]
#[non_exhaustive]
pub enum CoreEvent {
/// Speaking state update from the WS gateway, typically describing how another voice
/// user is transmitting audio data. Clients must send at least one such
/// packet to allow SSRC/UserID matching.
///
/// Fired on receipt of a speaking state update from another host.
///
/// Note: this will fire when a user starts speaking for the first time,
/// or changes their capabilities.
SpeakingStateUpdate,
#[cfg(feature = "receive")]
/// Fires every 20ms, containing the scheduled voice packet and decoded audio
/// data for each live user.
VoiceTick,
#[cfg(feature = "receive")]
/// Fires on receipt of a voice packet from another stream in the voice call.
///
/// As RTP packets do not map to Discord's notion of users, SSRCs must be mapped
/// back using the user IDs seen through client connection, disconnection,
/// or speaking state update.
RtpPacket,
#[cfg(feature = "receive")]
/// Fires on receipt of an RTCP packet, containing various call stats
/// such as latency reports.
RtcpPacket,
/// Fires whenever a user disconnects from the same stream as the bot.
ClientDisconnect,
/// Fires when this driver successfully connects to a voice channel.
DriverConnect,
/// Fires when this driver successfully reconnects after a network error.
DriverReconnect,
/// Fires when this driver fails to connect to, or drops from, a voice channel.
DriverDisconnect,
}

82
app/vendor/songbird/src/events/data.rs vendored Normal file
View File

@@ -0,0 +1,82 @@
use super::*;
use std::cmp::Ordering;
/// Internal representation of an event, as handled by the audio context.
pub struct EventData {
pub(crate) event: Event,
pub(crate) fire_time: Option<Duration>,
pub(crate) action: Box<dyn EventHandler>,
}
impl EventData {
/// Create a representation of an event and its associated handler.
///
/// An event handler, `action`, receives an [`EventContext`] and optionally
/// produces a new [`Event`] type for itself. Returning `None` will
/// maintain the same event type, while removing any [`Delayed`] entries.
/// Event handlers will be re-added with their new trigger condition,
/// or removed if [`Cancel`]led
///
/// [`EventContext`]: EventContext
/// [`Event`]: Event
/// [`Delayed`]: Event::Delayed
/// [`Cancel`]: Event::Cancel
pub fn new<F: EventHandler + 'static>(event: Event, action: F) -> Self {
Self {
event,
fire_time: None,
action: Box::new(action),
}
}
/// Computes the next firing time for a timer event.
pub fn compute_activation(&mut self, now: Duration) {
match self.event {
Event::Periodic(period, phase) => {
self.fire_time = Some(now + phase.unwrap_or(period));
},
Event::Delayed(offset) => {
self.fire_time = Some(now + offset);
},
_ => {},
}
}
}
impl std::fmt::Debug for EventData {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
write!(
f,
"Event {{ event: {:?}, fire_time: {:?}, action: <fn> }}",
self.event, self.fire_time
)
}
}
/// Events are ordered/compared based on their firing time.
impl Ord for EventData {
fn cmp(&self, other: &Self) -> Ordering {
// FIXME: we don't have let chains in this edition songbird uses so when we upgrade to 2024 edition,
// change this to a let chain that way it reads easier
if let Some(t1) = &self.fire_time {
if let Some(t2) = &other.fire_time {
return t2.cmp(t1);
}
}
Ordering::Equal
}
}
impl PartialOrd for EventData {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl PartialEq for EventData {
fn eq(&self, other: &Self) -> bool {
self.fire_time == other.fire_time
}
}
impl Eq for EventData {}

147
app/vendor/songbird/src/events/mod.rs vendored Normal file
View File

@@ -0,0 +1,147 @@
//! Events relating to tracks, timing, and other callers.
//!
//! ## Listening for events
//! Driver events in Songbird are composed of two parts:
//! * An [`Event`] to listen out for. These may be discrete events,
//! or generated by timers.
//! * An [`EventHandler`] to be called on receipt of an event. As event
//! handlers may be shared between several events, the handler is called
//! with an [`EventContext`] describing which event was fired.
//!
//! Event handlers are registered using functions such as [`Driver::add_global_event`],
//! or [`TrackHandle::add_event`], or. Internally, these pairs are stored
//! as [`EventData`].
//!
//! ## `EventHandler` lifecycle
//! An event handler is essentially just an async function which may return
//! another type of event to listen out for (an `Option<Event>`). For instance,
//! [`Some(Event::Cancel)`] will remove that event listener, while `None` won't
//! change it at all.
//!
//! The exception is one-off events like [`Event::Delayed`], which remove themselves
//! after one call *unless* an [`Event`] override is returned.
//!
//! ## Global and local listeners
//! *Global* event listeners are those which are placed onto the [`Driver`],
//! while *local* event listeners are those which are placed on a
//! [`Track`]/[`Handle`].
//!
//! Track or timed events, when local, return a reference to the parent track.
//! When registered globally, they fire on a per-tick basis, returning references to
//! all relevant tracks in that 20ms window. Global/local timed events use a global
//! timer or a [track's playback time], respectively.
//!
//! [`CoreEvent`]s may only be registered globally.
//!
//! [`Event`]: Event
//! [`EventHandler`]: EventHandler
//! [`EventContext`]: EventContext
//! [`Driver::add_global_event`]: crate::driver::Driver::add_global_event
//! [`Driver`]: crate::driver::Driver::add_global_event
//! [`TrackHandle::add_event`]: crate::tracks::TrackHandle::add_event
//! [`Track`]: crate::tracks::Track::events
//! [`Handle`]: crate::tracks::TrackHandle::add_event
//! [`EventData`]: EventData
//! [`Some(Event::Cancel)`]: Event::Cancel
//! [`Event::Delayed`]: Event::Delayed
//! [track's playback time]: crate::tracks::TrackState::play_time
//! [`CoreEvent`]: CoreEvent
mod context;
mod core;
mod data;
mod store;
mod track;
mod untimed;
pub use self::{
context::{context_data, EventContext},
core::*,
data::*,
store::*,
track::*,
untimed::*,
};
pub(crate) use context::{internal_data, CoreContext};
use async_trait::async_trait;
use std::time::Duration;
/// Trait to handle an event which can be fired per-track, or globally.
///
/// These may be feasibly reused between several event sources.
#[async_trait]
pub trait EventHandler: Send + Sync {
/// Respond to one received event.
async fn act(&self, ctx: &EventContext<'_>) -> Option<Event>;
}
/// Classes of event which may occur, triggering a handler
/// at the local (track-specific) or global level.
///
/// Local time-based events rely upon the current playback
/// time of a track, and so will not fire if a track becomes paused
/// or stops. In case this is required, global events are a better
/// fit.
///
/// Event handlers themselves are described in [`EventData::new`].
///
/// [`EventData::new`]: EventData::new
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
#[non_exhaustive]
pub enum Event {
/// Periodic events rely upon two parameters: a *period*
/// and an optional *phase*.
///
/// If the *phase* is `None`, then the event will first fire
/// in one *period*. Periodic events repeat automatically
/// so long as the `action` in [`EventData`] returns `None`.
///
/// [`EventData`]: EventData
Periodic(Duration, Option<Duration>),
/// Delayed events rely upon a *delay* parameter, and
/// fire one *delay* after the audio context processes them.
///
/// Delayed events are automatically removed once fired,
/// so long as the `action` in [`EventData`] returns `None`.
///
/// [`EventData`]: EventData
Delayed(Duration),
/// Track events correspond to certain actions or changes
/// of state, such as a track finishing, looping, or being
/// manually stopped.
///
/// Track events persist while the `action` in [`EventData`]
/// returns `None`.
///
/// [`EventData`]: EventData
Track(TrackEvent),
/// Core events
///
/// Track events persist while the `action` in [`EventData`]
/// returns `None`. Core events **must** be applied globally,
/// as attaching them to a track is a no-op.
///
/// [`EventData`]: EventData
Core(CoreEvent),
/// Cancels the event, if it was intended to persist.
Cancel,
}
impl Event {
pub(crate) fn is_global_only(&self) -> bool {
matches!(self, Self::Core(_))
}
}
impl From<TrackEvent> for Event {
fn from(evt: TrackEvent) -> Self {
Event::Track(evt)
}
}
impl From<CoreEvent> for Event {
fn from(evt: CoreEvent) -> Self {
Event::Core(evt)
}
}

263
app/vendor/songbird/src/events/store.rs vendored Normal file
View File

@@ -0,0 +1,263 @@
use super::*;
use crate::{
constants::*,
tracks::{ReadyState, TrackHandle, TrackState},
};
use enum_map::EnumMap;
use std::collections::{BinaryHeap, HashMap};
use tracing::info;
#[derive(Debug, Default)]
/// Storage for [`EventData`], designed to be used for both local and global contexts.
///
/// Timed events are stored in a binary heap for fast selection, and have custom `Eq`,
/// `Ord`, etc. implementations to support (only) this.
///
/// [`EventData`]: EventData
pub struct EventStore {
timed: BinaryHeap<EventData>,
untimed: Box<EnumMap<UntimedEvent, Vec<EventData>>>,
local_only: bool,
}
impl EventStore {
/// Creates a new event store to be used globally.
#[must_use]
pub fn new() -> Self {
Self::default()
}
/// Creates a new event store to be used within a [`Track`].
///
/// This is usually automatically installed by the driver once
/// a track has been registered.
///
/// [`Track`]: crate::tracks::Track
#[must_use]
pub fn new_local() -> Self {
EventStore {
local_only: true,
..Default::default()
}
}
/// Add an event to this store.
///
/// Updates `evt` according to [`EventData::compute_activation`].
///
/// [`EventData::compute_activation`]: EventData::compute_activation
pub fn add_event(&mut self, mut evt: EventData, now: Duration) {
evt.compute_activation(now);
if self.local_only && evt.event.is_global_only() {
return;
}
match evt.event {
Event::Core(c) => {
self.untimed[c.into()].push(evt);
},
Event::Track(t) => {
self.untimed[t.into()].push(evt);
},
Event::Delayed(_) | Event::Periodic(_, _) => {
self.timed.push(evt);
},
_ => {
// Event cancelled.
},
}
}
/// Processes all events due up to and including `now`.
pub(crate) async fn process_timed(&mut self, now: Duration, ctx: EventContext<'_>) {
while let Some(evt) = self.timed.peek() {
if evt
.fire_time
.as_ref()
.expect("Timed event must have a fire_time.")
> &now
{
break;
}
let mut evt = self
.timed
.pop()
.expect("Can only succeed due to peek = Some(...).");
let old_evt_type = evt.event;
if let Some(new_evt_type) = evt.action.act(&ctx).await {
evt.event = new_evt_type;
self.add_event(evt, now);
} else if let Event::Periodic(d, _) = old_evt_type {
evt.event = Event::Periodic(d, None);
self.add_event(evt, now);
}
}
}
/// Processes all events due up to and including `now`.
pub(crate) fn timed_event_ready(&self, now: Duration) -> bool {
self.timed.peek().is_some_and(|evt| {
evt.fire_time
.as_ref()
.expect("Timed event must have a fire_time.")
<= &now
})
}
/// Processes all events attached to the given track event.
pub(crate) async fn process_untimed(
&mut self,
now: Duration,
untimed_event: UntimedEvent,
ctx: EventContext<'_>,
) {
// move a Vec in and out: not too expensive, but could be better.
// Although it's obvious that moving an event out of one vec and into
// another necessitates that they be different event types, thus entries,
// convincing the compiler of this is non-trivial without making them dedicated
// fields.
let mut events = std::mem::take(&mut self.untimed[untimed_event]);
if !events.is_empty() {
// TODO: Possibly use tombstones to prevent realloc/memcpys?
// i.e., never shrink array, replace ended tracks with <DEAD>,
// maintain a "first-track" stack and freelist alongside.
let mut i = 0;
while i < events.len() {
let evt = &mut events[i];
// Only remove/readd if the event type changes (i.e., Some AND new != old)
if let Some(new_evt_type) = evt.action.act(&ctx).await {
if evt.event == new_evt_type {
let mut evt = events.remove(i);
evt.event = new_evt_type;
self.add_event(evt, now);
} else {
i += 1;
}
} else {
i += 1;
};
}
self.untimed[untimed_event] = events;
}
}
}
#[derive(Debug, Default)]
pub(crate) struct GlobalEvents {
pub(crate) store: EventStore,
pub(crate) time: Duration,
pub(crate) awaiting_tick: HashMap<TrackEvent, Vec<usize>>,
}
impl GlobalEvents {
pub(crate) fn add_event(&mut self, evt: EventData) {
self.store.add_event(evt, self.time);
}
pub(crate) async fn fire_core_event(&mut self, evt: CoreEvent, ctx: EventContext<'_>) {
self.store.process_untimed(self.time, evt.into(), ctx).await;
}
pub(crate) fn fire_track_event(&mut self, evt: TrackEvent, index: usize) {
let holder = self.awaiting_tick.entry(evt).or_default();
holder.push(index);
}
pub(crate) fn remove_handlers(&mut self) {
self.store = EventStore::new();
}
pub(crate) async fn tick(
&mut self,
events: &mut [EventStore],
states: &mut [TrackState],
handles: &mut [TrackHandle],
) {
// Global timed events
self.time += TIMESTEP_LENGTH;
if self.store.timed_event_ready(self.time) {
let global_ctx: Vec<(&TrackState, &TrackHandle)> =
states.iter().zip(handles.iter()).collect();
self.store
.process_timed(self.time, EventContext::Track(&global_ctx[..]))
.await;
}
// Local timed events
for (i, state) in states.iter_mut().enumerate() {
if state.playing.is_playing() && state.ready == ReadyState::Playable {
state.step_frame();
let event_store = events
.get_mut(i)
.expect("Missing store index for Tick (local timed).");
let handle = handles
.get_mut(i)
.expect("Missing handle index for Tick (local timed).");
event_store
.process_timed(state.play_time, EventContext::Track(&[(state, handle)]))
.await;
}
}
for (evt, indices) in &self.awaiting_tick {
let untimed = (*evt).into();
if !indices.is_empty() {
info!("Firing {:?} for {:?}", evt, indices);
}
// Local untimed track events.
for &i in indices {
let event_store = events
.get_mut(i)
.expect("Missing store index for Tick (local untimed).");
let handle = handles
.get_mut(i)
.expect("Missing handle index for Tick (local untimed).");
let state = states
.get_mut(i)
.expect("Missing state index for Tick (local untimed).");
event_store
.process_untimed(
state.position,
untimed,
EventContext::Track(&[(state, handle)]),
)
.await;
}
// Global untimed track events.
if !self.store.untimed[untimed].is_empty() && !indices.is_empty() {
let global_ctx: Vec<(&TrackState, &TrackHandle)> = indices
.iter()
.map(|i| {
(
states
.get(*i)
.expect("Missing state index for Tick (global untimed)"),
handles
.get(*i)
.expect("Missing handle index for Tick (global untimed)"),
)
})
.collect();
self.store
.process_untimed(self.time, untimed, EventContext::Track(&global_ctx[..]))
.await;
}
}
// Now drain vecs.
for indices in self.awaiting_tick.values_mut() {
indices.clear();
}
}
}

35
app/vendor/songbird/src/events/track.rs vendored Normal file
View File

@@ -0,0 +1,35 @@
// TODO: Could this be a bitset? Could accelerate lookups,
// allow easy joint subscription & remove Vecs for related evt handling?
/// Track events correspond to certain actions or changes
/// of state, such as a track finishing, looping, or being
/// manually stopped. Voice core events occur on receipt of
/// voice packets and telemetry.
///
/// Track events persist while the `action` in [`EventData`]
/// returns `None`.
///
/// [`EventData`]: super::EventData
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, enum_map::Enum)]
#[non_exhaustive]
pub enum TrackEvent {
/// The attached track has resumed playing.
///
/// This event will not fire when a track first starts,
/// but will fire when a track changes from, e.g., paused to playing.
/// This is most relevant for queue users: queued tracks placed into a
/// non-empty queue are initlally paused, and are later moved to `Play`.
Play,
/// The attached track has been paused.
Pause,
/// The attached track has ended.
End,
/// The attached track has looped.
Loop,
/// The attached track is being readied or recreated.
Preparing,
/// The attached track has become playable.
Playable,
/// The attached track has encountered a runtime or initialisation error.
Error,
}

View File

@@ -0,0 +1,29 @@
use super::*;
/// Track and voice core events.
///
/// Untimed events persist while the `action` in [`EventData`]
/// returns `None`.
///
/// [`EventData`]: EventData
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, enum_map::Enum)]
#[non_exhaustive]
pub enum UntimedEvent {
/// Untimed events belonging to a track, such as state changes, end, or loops.
Track(TrackEvent),
/// Untimed events belonging to the global context, such as finished tracks,
/// client speaking updates, or RT(C)P voice and telemetry data.
Core(CoreEvent),
}
impl From<TrackEvent> for UntimedEvent {
fn from(evt: TrackEvent) -> Self {
UntimedEvent::Track(evt)
}
}
impl From<CoreEvent> for UntimedEvent {
fn from(evt: CoreEvent) -> Self {
UntimedEvent::Core(evt)
}
}

View File

@@ -0,0 +1,64 @@
use nonmax::NonMaxU32;
use std::time::Duration;
/// A type similar to [`std::time::Duration`] except using an `f32`.
///
/// This limits precision, only allowing by-second precision up to ~194 days,
/// but in turn shinks type size from `16` bytes to `4` bytes.
///
/// # Implementation Note
///
/// We actually store this as `NonMaxU32`, in order to get a niche to make `Option<FloatDuration>` the same size.
#[derive(Clone, Copy)]
pub struct FloatDuration(NonMaxU32);
impl FloatDuration {
/// Creates a [`FloatDuration`] from a raw `f32` value.
///
/// # Panics
/// This may panic if `value` is a NaN value.
#[must_use]
pub fn from_secs_f32(value: f32) -> Self {
Self(NonMaxU32::new(value.to_bits()).expect("value should not be NaN"))
}
/// Converts a [`FloatDuration`] to an `f32`.
///
/// This is cheaper than converting to a [`Duration`] and using [`Duration::as_secs_f32`] as it is more direct.
#[must_use]
pub fn as_secs_f32(self) -> f32 {
f32::from_bits(self.0.get())
}
}
impl std::fmt::Debug for FloatDuration {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_tuple("FloatDuration")
.field(&Duration::from(*self))
.finish()
}
}
impl PartialEq for FloatDuration {
fn eq(&self, other: &Self) -> bool {
self.as_secs_f32() == other.as_secs_f32()
}
}
impl PartialOrd for FloatDuration {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
self.as_secs_f32().partial_cmp(&other.as_secs_f32())
}
}
impl From<Duration> for FloatDuration {
fn from(value: Duration) -> Self {
Self::from_secs_f32(value.as_secs_f32())
}
}
impl From<FloatDuration> for Duration {
fn from(value: FloatDuration) -> Self {
Duration::from_secs_f32(value.as_secs_f32())
}
}

490
app/vendor/songbird/src/handler.rs vendored Normal file
View File

@@ -0,0 +1,490 @@
#[cfg(feature = "driver")]
use crate::{driver::Driver, error::ConnectionResult};
use crate::{
error::{JoinError, JoinResult},
id::{ChannelId, GuildId, UserId},
info::{ConnectionInfo, ConnectionProgress},
join::*,
shards::{Shard, VoiceUpdate},
Config,
};
use flume::Sender;
use std::fmt::Debug;
use tracing::instrument;
#[cfg(feature = "driver")]
use std::ops::{Deref, DerefMut};
#[derive(Clone, Debug)]
enum Return {
// Return the connection info as it is received.
Info(Sender<ConnectionInfo>),
// Two channels: first indicates "gateway connection" was successful,
// second indicates that the driver successfully connected.
// The first is needed to cancel a timeout as the driver can/should
// have separate connection timing/retry config.
#[cfg(feature = "driver")]
Conn(Sender<()>, Sender<ConnectionResult<()>>),
}
/// The Call handler is responsible for a single voice connection, acting
/// as a clean API above the inner state and gateway message management.
///
/// If the `"driver"` feature is enabled, then a Call exposes all control methods of
/// [`Driver`] via `Deref(Mut)`.
///
/// [`Driver`]: struct@Driver
#[derive(Clone, Debug)]
pub struct Call {
#[cfg(not(feature = "driver"))]
config: Config,
connection: Option<(ConnectionProgress, Return)>,
#[cfg(feature = "driver")]
/// The internal controller of the voice connection monitor thread.
driver: Driver,
guild_id: GuildId,
/// Whether the current handler is set to deafen voice connections.
self_deaf: bool,
/// Whether the current handler is set to mute voice connections.
self_mute: bool,
user_id: UserId,
/// Will be set when a `Call` is made via the [`new`]
/// method.
///
/// When set via [`standalone`](`Call::standalone`), it will not be
/// present.
///
/// [`new`]: Call::new
/// [`standalone`]: Call::standalone
ws: Option<Shard>,
}
impl Call {
/// Creates a new Call, which will send out WebSocket messages via
/// the given shard.
#[inline]
#[instrument]
pub fn new<G, U>(guild_id: G, ws: Shard, user_id: U) -> Self
where
G: Into<GuildId> + Debug,
U: Into<UserId> + Debug,
{
Self::new_raw_cfg(guild_id.into(), Some(ws), user_id.into(), Config::default())
}
/// Creates a new Call, configuring the driver as specified.
#[inline]
#[instrument]
pub fn from_config<G, U>(guild_id: G, ws: Shard, user_id: U, config: Config) -> Self
where
G: Into<GuildId> + Debug,
U: Into<UserId> + Debug,
{
Self::new_raw_cfg(guild_id.into(), Some(ws), user_id.into(), config)
}
/// Creates a new, standalone Call which is not connected via
/// WebSocket to the Gateway.
///
/// Actions such as muting, deafening, and switching channels will not
/// function through this Call and must be done through some other
/// method, as the values will only be internally updated.
///
/// For most use cases you do not want this.
#[inline]
#[instrument]
pub fn standalone<G, U>(guild_id: G, user_id: U) -> Self
where
G: Into<GuildId> + Debug,
U: Into<UserId> + Debug,
{
Self::new_raw_cfg(guild_id.into(), None, user_id.into(), Config::default())
}
/// Creates a new standalone Call from the given configuration file.
#[inline]
#[instrument]
pub fn standalone_from_config<G, U>(guild_id: G, user_id: U, config: Config) -> Self
where
G: Into<GuildId> + Debug,
U: Into<UserId> + Debug,
{
Self::new_raw_cfg(guild_id.into(), None, user_id.into(), config)
}
fn new_raw_cfg(guild_id: GuildId, ws: Option<Shard>, user_id: UserId, config: Config) -> Self {
Call {
#[cfg(not(feature = "driver"))]
config,
connection: None,
#[cfg(feature = "driver")]
driver: Driver::new(config),
guild_id,
self_deaf: false,
self_mute: false,
user_id,
ws,
}
}
#[instrument(skip(self))]
fn do_connect(&mut self) {
match &self.connection {
Some((ConnectionProgress::Complete(c), Return::Info(tx))) => {
// It's okay if the receiver hung up.
drop(tx.send(c.clone()));
},
#[cfg(feature = "driver")]
Some((ConnectionProgress::Complete(c), Return::Conn(first_tx, driver_tx))) => {
// It's okay if the receiver hung up.
_ = first_tx.send(());
self.driver.raw_connect(c.clone(), driver_tx.clone());
},
_ => {},
}
}
/// Sets whether the current connection is to be deafened.
///
/// If there is no live voice connection, then this only acts as a settings
/// update for future connections.
///
/// **Note**: Unlike in the official client, you _can_ be deafened while
/// not being muted.
///
/// **Note**: If the `Call` was created via [`standalone`], then this
/// will _only_ update whether the connection is internally deafened.
///
/// [`standalone`]: Call::standalone
#[instrument(skip(self))]
pub async fn deafen(&mut self, deaf: bool) -> JoinResult<()> {
self.self_deaf = deaf;
self.update().await
}
/// Returns whether the current connection is self-deafened in this server.
///
/// This is purely cosmetic.
#[instrument(skip(self))]
pub fn is_deaf(&self) -> bool {
self.self_deaf
}
async fn should_actually_join<F, G>(
&mut self,
completion_generator: F,
tx: &Sender<G>,
channel_id: ChannelId,
) -> JoinResult<bool>
where
F: FnOnce(&Self) -> G,
{
Ok(if let Some(conn) = &self.connection {
if conn.0.in_progress() {
self.leave().await?;
true
} else if conn.0.channel_id() == channel_id {
drop(tx.send(completion_generator(self)));
false
} else {
// not in progress, and/or a channel change.
true
}
} else {
true
})
}
#[cfg(feature = "driver")]
/// Connect or switch to the given voice channel by its Id.
///
/// This function acts as a future in two stages:
/// * The first `await` sends the request over the gateway.
/// * The second `await`s a the driver's connection attempt.
/// To prevent deadlock, any mutexes around this Call
/// *must* be released before this result is queried.
///
/// When using [`Songbird::join`], this pattern is correctly handled for you.
///
/// [`Songbird::join`]: crate::Songbird::join
#[instrument(skip(self))]
#[inline]
pub async fn join<C>(&mut self, channel_id: C) -> JoinResult<Join>
where
C: Into<ChannelId> + Debug,
{
self.join_inner(channel_id.into()).await
}
#[cfg(feature = "driver")]
async fn join_inner(&mut self, channel_id: ChannelId) -> JoinResult<Join> {
let (tx, rx) = flume::unbounded();
let (gw_tx, gw_rx) = flume::unbounded();
let do_conn = self
.should_actually_join(|_| (), &gw_tx, channel_id)
.await?;
if do_conn {
self.connection = Some((
ConnectionProgress::new(self.guild_id, self.user_id, channel_id),
Return::Conn(gw_tx, tx),
));
let timeout = self.config().gateway_timeout.map(Into::into);
self.update()
.await
.map(|()| Join::new(rx.into_recv_async(), gw_rx.into_recv_async(), timeout))
} else {
// Skipping the gateway connection implies that the current connection is complete
// AND the channel is a match.
//
// Send a polite request to the driver, which should only *actually* reconnect
// if it had a problem earlier.
let info = self.current_connection().unwrap().clone();
self.driver.raw_connect(info, tx.clone());
Ok(Join::new(
rx.into_recv_async(),
gw_rx.into_recv_async(),
None,
))
}
}
/// Join the selected voice channel, *without* running/starting an RTP
/// session or running the driver.
///
/// Use this if you require connection info for lavalink,
/// some other voice implementation, or don't want to use the driver for a given call.
///
/// This function acts as a future in two stages:
/// * The first `await` sends the request over the gateway.
/// * The second `await`s voice session data from Discord.
/// To prevent deadlock, any mutexes around this Call
/// *must* be released before this result is queried.
///
/// When using [`Songbird::join_gateway`], this pattern is correctly handled for you.
///
/// [`Songbird::join_gateway`]: crate::Songbird::join_gateway
#[instrument(skip(self))]
#[inline]
pub async fn join_gateway<C>(&mut self, channel_id: C) -> JoinResult<JoinGateway>
where
C: Into<ChannelId> + Debug,
{
self.join_gateway_inner(channel_id.into()).await
}
async fn join_gateway_inner(&mut self, channel_id: ChannelId) -> JoinResult<JoinGateway> {
let (tx, rx) = flume::unbounded();
let do_conn = self
.should_actually_join(
|call| call.connection.as_ref().unwrap().0.info().unwrap(),
&tx,
channel_id,
)
.await?;
if do_conn {
self.connection = Some((
ConnectionProgress::new(self.guild_id, self.user_id, channel_id),
Return::Info(tx),
));
let timeout = self.config().gateway_timeout.map(Into::into);
self.update()
.await
.map(|()| JoinGateway::new(rx.into_recv_async(), timeout))
} else {
Ok(JoinGateway::new(rx.into_recv_async(), None))
}
}
/// Returns the current voice connection details for this Call,
/// if available.
#[instrument(skip(self))]
pub fn current_connection(&self) -> Option<&ConnectionInfo> {
match &self.connection {
Some((progress, _)) => progress.get_connection_info(),
_ => None,
}
}
/// Returns `id` of the channel, if connected or connecting to any.
///
/// This remains set after a connection failure, to allow for reconnection
/// as needed. This will change if moved into another voice channel by an
/// admin, and will be unset if kicked from a voice channel.
#[instrument(skip(self))]
pub fn current_channel(&self) -> Option<ChannelId> {
match &self.connection {
Some((progress, _)) => Some(progress.channel_id()),
_ => None,
}
}
/// Leaves the current voice channel, disconnecting from it.
///
/// This does _not_ forget settings, like whether to be self-deafened or
/// self-muted.
///
/// **Note**: If the `Call` was created via [`standalone`], then this
/// will _only_ update whether the connection is internally connected to a
/// voice channel.
///
/// [`standalone`]: Call::standalone
#[instrument(skip(self))]
pub async fn leave(&mut self) -> JoinResult<()> {
self.leave_local();
// Only send an update if we were in a voice channel.
self.update().await
}
fn leave_local(&mut self) {
self.connection = None;
#[cfg(feature = "driver")]
self.driver.leave();
}
/// Sets whether the current connection is to be muted.
///
/// If there is no live voice connection, then this only acts as a settings
/// update for future connections.
///
/// **Note**: If the `Call` was created via [`standalone`], then this
/// will _only_ update whether the connection is internally muted.
///
/// [`standalone`]: Call::standalone
#[instrument(skip(self))]
pub async fn mute(&mut self, mute: bool) -> JoinResult<()> {
self.self_mute = mute;
#[cfg(feature = "driver")]
self.driver.mute(mute);
self.update().await
}
/// Returns whether the current connection is self-muted in this server.
#[instrument(skip(self))]
pub fn is_mute(&self) -> bool {
self.self_mute
}
/// Updates the voice server data.
///
/// You should only need to use this if you initialized the `Call` via
/// [`standalone`].
///
/// [`standalone`]: Call::standalone
#[instrument(skip(self, token))]
pub fn update_server(&mut self, endpoint: String, token: String) {
let try_conn = if let Some((ref mut progress, _)) = self.connection.as_mut() {
progress.apply_server_update(endpoint, token)
} else {
false
};
if try_conn {
self.do_connect();
}
}
/// Updates the internal voice state of the current user.
///
/// You should only need to use this if you initialized the `Call` via
/// [`standalone`].
///
/// [`standalone`]: Call::standalone
#[instrument(skip(self))]
#[inline]
pub fn update_state<C>(&mut self, session_id: String, channel_id: Option<C>)
where
C: Into<ChannelId> + Debug,
{
self.update_state_inner(session_id, channel_id.map(Into::into));
}
fn update_state_inner(&mut self, session_id: String, channel_id: Option<ChannelId>) {
if let Some(channel_id) = channel_id {
let try_conn = if let Some((ref mut progress, _)) = self.connection.as_mut() {
progress.apply_state_update(session_id, channel_id)
} else {
false
};
if try_conn {
self.do_connect();
}
} else {
// Likely that we were disconnected by an admin.
self.leave_local();
}
}
/// Send an update for the current session over WS.
///
/// Does nothing if initialized via [`standalone`].
///
/// [`standalone`]: Call::standalone
#[instrument(skip(self))]
async fn update(&mut self) -> JoinResult<()> {
if let Some(ws) = self.ws.as_mut() {
ws.update_voice_state(
self.guild_id,
self.connection.as_ref().map(|c| c.0.channel_id()),
self.self_deaf,
self.self_mute,
)
.await
} else {
Err(JoinError::NoSender)
}
}
}
#[cfg(not(feature = "driver"))]
impl Call {
/// Access this call handler's configuration.
pub fn config(&self) -> &Config {
&self.config
}
/// Mutably access this call handler's configuration.
pub fn config_mut(&mut self) -> &mut Config {
&mut self.config
}
/// Set this call handler's configuration.
pub fn set_config(&mut self, config: Config) {
self.config = config;
}
}
#[cfg(feature = "driver")]
impl Deref for Call {
type Target = Driver;
fn deref(&self) -> &Self::Target {
&self.driver
}
}
#[cfg(feature = "driver")]
impl DerefMut for Call {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.driver
}
}

123
app/vendor/songbird/src/id.rs vendored Normal file
View File

@@ -0,0 +1,123 @@
//! Newtypes around Discord IDs for library cross-compatibility.
#[cfg(feature = "driver")]
use crate::model::id::{GuildId as DriverGuild, UserId as DriverUser};
#[cfg(feature = "serenity")]
use serenity::model::id::{
ChannelId as SerenityChannel,
GuildId as SerenityGuild,
UserId as SerenityUser,
};
use std::{
fmt::{Display, Formatter, Result as FmtResult},
num::NonZeroU64,
};
#[cfg(feature = "twilight")]
use twilight_model::id::{
marker::{ChannelMarker, GuildMarker, UserMarker},
Id as TwilightId,
};
/// ID of a Discord voice/text channel.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct ChannelId(pub NonZeroU64);
/// ID of a Discord guild (colloquially, "server").
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct GuildId(pub NonZeroU64);
/// ID of a Discord user.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct UserId(pub NonZeroU64);
impl Display for ChannelId {
fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult {
Display::fmt(&self.0, f)
}
}
impl From<NonZeroU64> for ChannelId {
fn from(id: NonZeroU64) -> Self {
Self(id)
}
}
#[cfg(feature = "serenity")]
impl From<SerenityChannel> for ChannelId {
fn from(id: SerenityChannel) -> Self {
Self(NonZeroU64::new(id.get()).unwrap())
}
}
#[cfg(feature = "twilight")]
impl From<TwilightId<ChannelMarker>> for ChannelId {
fn from(id: TwilightId<ChannelMarker>) -> Self {
Self(id.into_nonzero())
}
}
impl Display for GuildId {
fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult {
Display::fmt(&self.0, f)
}
}
impl From<NonZeroU64> for GuildId {
fn from(id: NonZeroU64) -> Self {
Self(id)
}
}
#[cfg(feature = "serenity")]
impl From<SerenityGuild> for GuildId {
fn from(id: SerenityGuild) -> Self {
Self(NonZeroU64::new(id.get()).unwrap())
}
}
#[cfg(feature = "driver")]
impl From<GuildId> for DriverGuild {
fn from(id: GuildId) -> Self {
Self(id.0.get())
}
}
#[cfg(feature = "twilight")]
impl From<TwilightId<GuildMarker>> for GuildId {
fn from(id: TwilightId<GuildMarker>) -> Self {
Self(id.into_nonzero())
}
}
impl Display for UserId {
fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult {
Display::fmt(&self.0, f)
}
}
impl From<NonZeroU64> for UserId {
fn from(id: NonZeroU64) -> Self {
Self(id)
}
}
#[cfg(feature = "serenity")]
impl From<SerenityUser> for UserId {
fn from(id: SerenityUser) -> Self {
Self(NonZeroU64::new(id.get()).unwrap())
}
}
#[cfg(feature = "driver")]
impl From<UserId> for DriverUser {
fn from(id: UserId) -> Self {
Self(id.0.get())
}
}
#[cfg(feature = "twilight")]
impl From<TwilightId<UserMarker>> for UserId {
fn from(id: TwilightId<UserMarker>) -> Self {
Self(id.into_nonzero())
}
}

198
app/vendor/songbird/src/info.rs vendored Normal file
View File

@@ -0,0 +1,198 @@
use crate::id::{ChannelId, GuildId, UserId};
use std::fmt;
#[derive(Clone, Debug)]
pub(crate) enum ConnectionProgress {
Complete(ConnectionInfo),
Incomplete(Partial),
}
impl ConnectionProgress {
pub(crate) fn new(guild_id: GuildId, user_id: UserId, channel_id: ChannelId) -> Self {
ConnectionProgress::Incomplete(Partial {
channel_id,
guild_id,
user_id,
token: None,
endpoint: None,
session_id: None,
})
}
pub(crate) fn get_connection_info(&self) -> Option<&ConnectionInfo> {
if let Self::Complete(c) = self {
Some(c)
} else {
None
}
}
pub(crate) fn in_progress(&self) -> bool {
matches!(self, ConnectionProgress::Incomplete(_))
}
pub(crate) fn channel_id(&self) -> ChannelId {
match self {
ConnectionProgress::Complete(conn_info) => conn_info.channel_id,
ConnectionProgress::Incomplete(part) => part.channel_id,
}
}
pub(crate) fn guild_id(&self) -> GuildId {
match self {
ConnectionProgress::Complete(conn_info) => conn_info.guild_id,
ConnectionProgress::Incomplete(part) => part.guild_id,
}
}
pub(crate) fn user_id(&self) -> UserId {
match self {
ConnectionProgress::Complete(conn_info) => conn_info.user_id,
ConnectionProgress::Incomplete(part) => part.user_id,
}
}
pub(crate) fn info(&self) -> Option<ConnectionInfo> {
self.get_connection_info().cloned()
}
pub(crate) fn apply_state_update(&mut self, session_id: String, channel_id: ChannelId) -> bool {
if self.channel_id() != channel_id {
// Likely that the bot was moved to a different channel by an admin.
*self = ConnectionProgress::new(self.guild_id(), self.user_id(), channel_id);
}
match self {
Self::Complete(c) => {
let should_reconn = c.session_id != session_id;
c.session_id = session_id;
should_reconn
},
Self::Incomplete(i) => i
.apply_state_update(session_id, channel_id)
.map(|info| {
*self = Self::Complete(info);
})
.is_some(),
}
}
pub(crate) fn apply_server_update(&mut self, endpoint: String, token: String) -> bool {
match self {
Self::Complete(c) => {
let should_reconn = c.endpoint != endpoint || c.token != token;
c.endpoint = endpoint;
c.token = token;
should_reconn
},
Self::Incomplete(i) => i
.apply_server_update(endpoint, token)
.map(|info| {
*self = Self::Complete(info);
})
.is_some(),
}
}
}
/// Parameters and information needed to start communicating with Discord's voice servers, either
/// with the Songbird driver, lavalink, or other system.
#[derive(Clone, Eq, Hash, PartialEq)]
pub struct ConnectionInfo {
/// ID of the voice channel being joined.
pub channel_id: ChannelId,
/// URL of the voice websocket gateway server assigned to this call.
pub endpoint: String,
/// ID of the target voice channel's parent guild.
///
/// Bots cannot connect to a guildless (i.e., direct message) voice call.
pub guild_id: GuildId,
/// Unique string describing this session for validation/authentication purposes.
pub session_id: String,
/// Ephemeral secret used to validate the above session.
pub token: String,
/// UserID of this bot.
pub user_id: UserId,
}
impl fmt::Debug for ConnectionInfo {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ConnectionInfo")
.field("channel_id", &self.channel_id)
.field("endpoint", &self.endpoint)
.field("guild_id", &self.guild_id)
.field("session_id", &self.session_id)
.field("token", &"<secret>")
.field("user_id", &self.user_id)
.finish()
}
}
#[derive(Clone)]
pub(crate) struct Partial {
pub channel_id: ChannelId,
pub endpoint: Option<String>,
pub guild_id: GuildId,
pub session_id: Option<String>,
pub token: Option<String>,
pub user_id: UserId,
}
impl fmt::Debug for Partial {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Partial")
.field("channel_id", &self.channel_id)
.field("endpoint", &self.endpoint)
.field("guild_id", &self.guild_id)
.field("session_id", &self.session_id)
.field("token_is_some", &self.token.is_some())
.field("user_id", &self.user_id)
.finish()
}
}
impl Partial {
fn finalise(&mut self) -> Option<ConnectionInfo> {
if self.endpoint.is_some() && self.session_id.is_some() && self.token.is_some() {
let endpoint = self.endpoint.take().unwrap();
let session_id = self.session_id.take().unwrap();
let token = self.token.take().unwrap();
Some(ConnectionInfo {
channel_id: self.channel_id,
endpoint,
session_id,
token,
guild_id: self.guild_id,
user_id: self.user_id,
})
} else {
None
}
}
fn apply_state_update(
&mut self,
session_id: String,
channel_id: ChannelId,
) -> Option<ConnectionInfo> {
if self.channel_id != channel_id {
self.endpoint = None;
self.token = None;
}
self.channel_id = channel_id;
self.session_id = Some(session_id);
self.finalise()
}
fn apply_server_update(&mut self, endpoint: String, token: String) -> Option<ConnectionInfo> {
self.endpoint = Some(endpoint);
self.token = Some(token);
self.finalise()
}
}

View File

@@ -0,0 +1,385 @@
use crate::input::AudioStreamError;
use async_trait::async_trait;
use flume::{Receiver, RecvError, Sender, TryRecvError};
use futures::{future::Either, stream::FuturesUnordered, FutureExt, StreamExt};
use parking_lot::Mutex;
use ringbuf::{storage::Heap, traits::*, *};
use std::{
io::{
Error as IoError,
ErrorKind as IoErrorKind,
Read,
Result as IoResult,
Seek,
SeekFrom,
Write,
},
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
};
use symphonia_core::io::MediaSource;
use tokio::{
io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt},
sync::Notify,
};
struct AsyncAdapterSink {
bytes_in: HeapProd<u8>,
req_rx: Receiver<AdapterRequest>,
resp_tx: Sender<AdapterResponse>,
stream: Box<dyn AsyncMediaSource>,
notify_rx: Arc<Notify>,
}
impl AsyncAdapterSink {
async fn launch(mut self) {
let mut inner_buf = vec![0u8; 32 * 1024].into_boxed_slice();
let mut read_region = 0..0;
let mut hit_end = false;
let mut blocked = false;
let mut pause_buf_moves = false;
let mut seek_res = None;
let mut seen_bytes = 0;
loop {
// if read_region is empty, refill from src.
// if that read is zero, tell other half.
// if WouldBlock, block on msg acquire,
// else non_block msg acquire.
if !pause_buf_moves {
if !hit_end && read_region.is_empty() {
if let Ok(n) = self.stream.read(&mut inner_buf).await {
read_region = 0..n;
if n == 0 {
drop(self.resp_tx.send_async(AdapterResponse::ReadZero).await);
hit_end = true;
}
seen_bytes += n as u64;
} else {
match self.stream.try_resume(seen_bytes).await {
Ok(s) => {
self.stream = s;
},
Err(_e) => break,
}
}
}
while !read_region.is_empty() && !blocked {
if let Ok(n_moved) = self
.bytes_in
.write(&inner_buf[read_region.start..read_region.end])
{
read_region.start += n_moved;
drop(self.resp_tx.send_async(AdapterResponse::ReadOccurred).await);
} else {
blocked = true;
}
}
}
let msg = if blocked || hit_end {
let mut fs = FuturesUnordered::new();
fs.push(Either::Left(self.req_rx.recv_async()));
fs.push(Either::Right(self.notify_rx.notified().map(|()| {
let o: Result<AdapterRequest, RecvError> = Ok(AdapterRequest::Wake);
o
})));
match fs.next().await {
Some(Ok(a)) => a,
_ => break,
}
} else {
match self.req_rx.try_recv() {
Ok(a) => a,
Err(TryRecvError::Empty) => continue,
_ => break,
}
};
match msg {
AdapterRequest::Wake => blocked = false,
AdapterRequest::ByteLen => {
drop(
self.resp_tx
.send_async(AdapterResponse::ByteLen(self.stream.byte_len().await))
.await,
);
},
AdapterRequest::Seek(pos) => {
pause_buf_moves = true;
drop(self.resp_tx.send_async(AdapterResponse::SeekClear).await);
seek_res = Some(self.stream.seek(pos).await);
},
AdapterRequest::SeekCleared => {
if let Some(res) = seek_res.take() {
drop(
self.resp_tx
.send_async(AdapterResponse::SeekResult(res))
.await,
);
}
pause_buf_moves = false;
},
}
}
}
}
/// An adapter for converting an async media source into a synchronous one
/// usable by symphonia.
///
/// This adapter takes a source implementing `AsyncRead`, and allows the receive side to
/// pass along seek requests needed. This allows for passing bytes from exclusively `AsyncRead`
/// streams (e.g., hyper HTTP sessions) to Songbird.
pub struct AsyncAdapterStream {
// Note: this mutex is here to appease symphonia's Send + Sync bound.
// Only one thread should own and pull from this stream, so in practice
// there is no contention.
bytes_out: Mutex<HeapCons<u8>>,
can_seek: bool,
// Note: these are Atomic just to work around the need for
// check_messages to take &self rather than &mut.
finalised: AtomicBool,
bytes_known_present: AtomicBool,
req_tx: Sender<AdapterRequest>,
resp_rx: Receiver<AdapterResponse>,
notify_tx: Arc<Notify>,
}
impl AsyncAdapterStream {
/// Wrap and pull from an async file stream, with an intermediate ring-buffer of size `buf_len`
/// between the async and sync halves.
#[must_use]
pub fn new(stream: Box<dyn AsyncMediaSource>, buf_len: usize) -> AsyncAdapterStream {
let (bytes_in, bytes_out) = SharedRb::<Heap<_>>::new(buf_len).split();
let bytes_out = bytes_out.into();
let (resp_tx, resp_rx) = flume::unbounded();
let (req_tx, req_rx) = flume::unbounded();
let can_seek = stream.is_seekable();
let notify_rx = Arc::new(Notify::new());
let notify_tx = notify_rx.clone();
let sink = AsyncAdapterSink {
bytes_in,
req_rx,
resp_tx,
stream,
notify_rx,
};
let stream = AsyncAdapterStream {
bytes_out,
can_seek,
finalised: false.into(),
bytes_known_present: false.into(),
req_tx,
resp_rx,
notify_tx,
};
tokio::spawn(async move {
Box::pin(sink.launch()).await;
});
stream
}
fn handle_messages(&self, op: Operation) -> Option<AdapterResponse> {
loop {
let msg = if op.will_block() {
self.resp_rx.recv().ok()
} else {
self.resp_rx.try_recv().ok()
};
let Some(msg) = msg else { break None };
// state changes
match &msg {
AdapterResponse::ReadZero => {
self.finalised.store(true, Ordering::Relaxed);
},
AdapterResponse::ReadOccurred => {
self.bytes_known_present.store(true, Ordering::Relaxed);
},
_ => {},
}
if op.expected_msg(&msg) {
break Some(msg);
}
}
}
fn is_dropped_and_clear(&self) -> bool {
self.resp_rx.is_empty() && self.resp_rx.is_disconnected()
}
fn check_dropped(&self) -> IoResult<()> {
if self.is_dropped_and_clear() {
Err(IoError::new(
IoErrorKind::UnexpectedEof,
"Async half was dropped.",
))
} else {
Ok(())
}
}
}
impl Read for AsyncAdapterStream {
fn read(&mut self, buf: &mut [u8]) -> IoResult<usize> {
loop {
let block = !(self.bytes_known_present.load(Ordering::Relaxed)
|| self.finalised.load(Ordering::Relaxed));
drop(self.handle_messages(Operation::Read { block }));
let mut rb = self.bytes_out.lock();
match rb.read(buf) {
Ok(n) => {
self.notify_tx.notify_one();
return Ok(n);
},
Err(e) if e.kind() == IoErrorKind::WouldBlock => {
// receive side must ABSOLUTELY be unblocked here.
self.notify_tx.notify_one();
if self.finalised.load(Ordering::Relaxed) {
return Ok(0);
}
self.bytes_known_present.store(false, Ordering::Relaxed);
self.check_dropped()?;
},
a => {
println!("Misc err {a:?}");
return a;
},
}
}
}
}
impl Seek for AsyncAdapterStream {
fn seek(&mut self, pos: SeekFrom) -> IoResult<u64> {
if !self.can_seek {
return Err(IoError::new(
IoErrorKind::Unsupported,
"Async half does not support seek operations.",
));
}
self.check_dropped()?;
_ = self.req_tx.send(AdapterRequest::Seek(pos));
// wait for async to tell us that it has stopped writing,
// then clear buf and allow async to write again.
self.finalised.store(false, Ordering::Relaxed);
match self.handle_messages(Operation::Seek) {
Some(AdapterResponse::SeekClear) => {},
None => self.check_dropped().map(|()| unreachable!())?,
_ => unreachable!(),
}
let mut rb = self.bytes_out.lock();
let cap = rb.capacity();
rb.skip(cap.into());
_ = self.req_tx.send(AdapterRequest::SeekCleared);
match self.handle_messages(Operation::Seek) {
Some(AdapterResponse::SeekResult(a)) => a,
None => self.check_dropped().map(|()| unreachable!()),
_ => unreachable!(),
}
}
}
impl MediaSource for AsyncAdapterStream {
fn is_seekable(&self) -> bool {
self.can_seek
}
fn byte_len(&self) -> Option<u64> {
self.check_dropped().ok()?;
_ = self.req_tx.send(AdapterRequest::ByteLen);
match self.handle_messages(Operation::Len) {
Some(AdapterResponse::ByteLen(a)) => a,
None => self.check_dropped().ok().map(|()| unreachable!()),
_ => unreachable!(),
}
}
}
enum AdapterRequest {
Wake,
Seek(SeekFrom),
SeekCleared,
ByteLen,
}
enum AdapterResponse {
SeekResult(IoResult<u64>),
SeekClear,
ByteLen(Option<u64>),
ReadZero,
ReadOccurred,
}
#[derive(Copy, Clone)]
enum Operation {
Read { block: bool },
Seek,
Len,
}
impl Operation {
fn will_block(self) -> bool {
match self {
Self::Read { block } => block,
_ => true,
}
}
fn expected_msg(self, msg: &AdapterResponse) -> bool {
match self {
Self::Read { .. } => matches!(
msg,
AdapterResponse::ReadOccurred | AdapterResponse::ReadZero
),
Self::Seek => matches!(
msg,
AdapterResponse::SeekResult(_) | AdapterResponse::SeekClear
),
Self::Len => matches!(msg, AdapterResponse::ByteLen(_)),
}
}
}
/// An async port of symphonia's [`MediaSource`].
///
/// Streams which are not seekable should implement `AsyncSeek` such that all operations
/// fail with `Unsupported`, and implement `fn is_seekable(&self) -> { false }`.
///
/// [`MediaSource`]: MediaSource
#[async_trait]
pub trait AsyncMediaSource: AsyncRead + AsyncSeek + Send + Sync + Unpin {
/// Returns if the source is seekable. This may be an expensive operation.
fn is_seekable(&self) -> bool;
/// Returns the length in bytes, if available. This may be an expensive operation.
async fn byte_len(&self) -> Option<u64>;
/// Tries to recreate this stream in event of an error, resuming from the given offset.
async fn try_resume(
&mut self,
_offset: u64,
) -> Result<Box<dyn AsyncMediaSource>, AudioStreamError> {
Err(AudioStreamError::Unsupported)
}
}

View File

@@ -0,0 +1,526 @@
use super::{compressed_cost_per_sec, default_config, CodecCacheError, ToAudioBytes};
use crate::{
constants::*,
input::{
codecs::{dca::*, get_codec_registry, get_probe},
AudioStream,
Input,
LiveInput,
},
};
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use opus2::{Application, Bitrate, Channels, Encoder as OpusEncoder, ErrorCode as OpusErrorCode};
use std::{
io::{
Cursor,
Error as IoError,
ErrorKind as IoErrorKind,
Read,
Result as IoResult,
Seek,
SeekFrom,
},
mem,
sync::atomic::{AtomicUsize, Ordering},
};
use streamcatcher::{
Config as ScConfig,
NeedsBytes,
Stateful,
Transform,
TransformPosition,
TxCatcher,
};
use symphonia_core::{
audio::Channels as SChannels,
codecs::CodecRegistry,
io::MediaSource,
meta::{MetadataRevision, StandardTagKey, Value},
probe::{Probe, ProbedMetadata},
};
use tracing::{debug, trace};
/// Configuration for a cached source.
pub struct Config {
/// Registry of audio codecs supported by the driver.
///
/// Defaults to [`get_codec_registry`], which adds opus2-based Opus codec support
/// to all of Symphonia's default codecs.
pub codec_registry: &'static CodecRegistry,
/// Registry of the muxers and container formats supported by the driver.
///
/// Defaults to [`get_probe`], which includes all of Symphonia's default format handlers
/// and DCA format support.
pub format_registry: &'static Probe,
/// Configuration for the inner streamcatcher instance.
///
/// Notably, this governs size hints and resize logic.
pub streamcatcher: ScConfig,
}
impl Default for Config {
fn default() -> Self {
Self {
codec_registry: get_codec_registry(),
format_registry: get_probe(),
streamcatcher: ScConfig::default(),
}
}
}
impl Config {
/// Generate a storage configuration given an estimated storage bitrate
/// `cost_per_sec` in bytes/s.
#[must_use]
pub fn default_from_cost(cost_per_sec: usize) -> Self {
let streamcatcher = default_config(cost_per_sec);
Self {
streamcatcher,
..Default::default()
}
}
}
/// A wrapper around an existing [`Input`] which compresses
/// the input using the Opus codec before storing it in memory.
///
/// The main purpose of this wrapper is to enable seeking on
/// incompatible sources and to ease resource consumption for
/// commonly reused/shared tracks. If only one Opus-compressed track
/// is playing at a time, then this removes the runtime decode cost
/// from the driver.
///
/// This is intended for use with larger, repeatedly used audio
/// tracks shared between sources, and stores the sound data
/// retrieved as **compressed Opus audio**.
///
/// Internally, this stores the stream and its metadata as a DCA1 file,
/// which can be written out to disk for later use.
///
/// [`Input`]: crate::input::Input
#[derive(Clone)]
pub struct Compressed {
/// Inner shared bytestore.
pub raw: TxCatcher<ToAudioBytes, OpusCompressor>,
}
impl Compressed {
/// Wrap an existing [`Input`] with an in-memory store, compressed using Opus.
///
/// [`Input`]: Input
pub async fn new(source: Input, bitrate: Bitrate) -> Result<Self, CodecCacheError> {
Self::with_config(source, bitrate, None).await
}
/// Wrap an existing [`Input`] with an in-memory store, compressed using Opus, with
/// custom configuration for both Symphonia and the backing store.
///
/// [`Input`]: Input
pub async fn with_config(
source: Input,
bitrate: Bitrate,
config: Option<Config>,
) -> Result<Self, CodecCacheError> {
let input = match source {
Input::Lazy(mut r) => {
let created = if r.should_create_async() {
r.create_async().await.map_err(CodecCacheError::from)
} else {
tokio::task::spawn_blocking(move || r.create().map_err(CodecCacheError::from))
.await
.map_err(CodecCacheError::from)
.and_then(|v| v)
};
created.map(LiveInput::Raw)
},
Input::Live(LiveInput::Parsed(_), _) => Err(CodecCacheError::StreamNotAtStart),
Input::Live(a, _rec) => Ok(a),
}?;
let cost_per_sec = compressed_cost_per_sec(bitrate);
let config = config.unwrap_or_else(|| Config::default_from_cost(cost_per_sec));
let promoted = tokio::task::spawn_blocking(move || {
input.promote(config.codec_registry, config.format_registry)
})
.await??;
// If success, guaranteed to be Parsed
let LiveInput::Parsed(mut parsed) = promoted else {
unreachable!()
};
// TODO: apply length hint.
// if config.length_hint.is_none() {
// if let Some(dur) = metadata.duration {
// apply_length_hint(&mut config, dur, cost_per_sec);
// }
// }
let track_info = parsed.decoder.codec_params();
let chan_count = track_info.channels.map_or(2, SChannels::count);
let (channels, stereo) = if chan_count >= 2 {
(Channels::Stereo, true)
} else {
(Channels::Mono, false)
};
let mut encoder = OpusEncoder::new(48000, channels, Application::Audio)?;
encoder.set_bitrate(bitrate)?;
let codec_type = parsed.decoder.codec_params().codec;
let encoding = config
.codec_registry
.get_codec(codec_type)
.map(|v| v.short_name.to_string());
let format_meta_hold = parsed.format.metadata();
let format_meta = format_meta_hold.current();
let metadata = create_metadata(
&mut parsed.meta,
format_meta,
&mut encoder,
chan_count as u8,
encoding,
)?;
let mut metabytes = b"DCA1\0\0\0\0".to_vec();
let orig_len = metabytes.len();
serde_json::to_writer(&mut metabytes, &metadata)?;
let meta_len = (metabytes.len() - orig_len)
.try_into()
.map_err(|_| CodecCacheError::MetadataTooLarge)?;
(&mut metabytes[4..][..mem::size_of::<i32>()])
.write_i32::<LittleEndian>(meta_len)
.expect("Magic byte writing location guaranteed to be well-founded.");
let source = ToAudioBytes::new(parsed, Some(2));
let raw = config
.streamcatcher
.build_tx(source, OpusCompressor::new(encoder, stereo, metabytes))?;
Ok(Self { raw })
}
/// Acquire a new handle to this object, creating a new
/// view of the existing cached data from the beginning.
#[must_use]
pub fn new_handle(&self) -> Self {
Self {
raw: self.raw.new_handle(),
}
}
}
fn create_metadata(
probe_metadata: &mut ProbedMetadata,
track_metadata: Option<&MetadataRevision>,
opus: &mut OpusEncoder,
channels: u8,
encoding: Option<String>,
) -> Result<DcaMetadata, CodecCacheError> {
let dca = DcaInfo {
version: 1,
tool: Tool {
name: env!("CARGO_PKG_NAME").into(),
version: env!("CARGO_PKG_VERSION").into(),
url: Some(env!("CARGO_PKG_HOMEPAGE").into()),
author: Some(env!("CARGO_PKG_AUTHORS").into()),
},
};
let abr = match opus.get_bitrate()? {
Bitrate::Bits(i) => Some(i as u64),
Bitrate::Auto => None,
Bitrate::Max => Some(510_000),
};
let mode = match opus.get_application()? {
Application::Voip => "voip",
Application::Audio => "music",
Application::LowDelay => "lowdelay",
}
.to_string();
let sample_rate = opus.get_sample_rate()?;
let opus = Opus {
mode,
sample_rate,
frame_size: MONO_FRAME_BYTE_SIZE as u64,
abr,
vbr: opus.get_vbr()?,
channels: channels.min(2),
};
let mut origin = Origin {
source: Some("file".into()),
abr: None,
channels: Some(channels),
encoding,
url: None,
};
let mut info = Info {
title: None,
artist: None,
album: None,
genre: None,
cover: None,
comments: None,
};
if let Some(meta) = probe_metadata.get() {
apply_meta_to_dca(&mut info, &mut origin, meta.current());
}
apply_meta_to_dca(&mut info, &mut origin, track_metadata);
Ok(DcaMetadata {
dca,
opus,
info: Some(info),
origin: Some(origin),
extra: None,
})
}
fn apply_meta_to_dca(info: &mut Info, origin: &mut Origin, src_meta: Option<&MetadataRevision>) {
if let Some(meta) = src_meta {
for tag in meta.tags() {
match tag.std_key {
Some(StandardTagKey::Album) =>
if let Value::String(s) = &tag.value {
info.album = Some(s.clone());
},
Some(StandardTagKey::Artist) =>
if let Value::String(s) = &tag.value {
info.artist = Some(s.clone());
},
Some(StandardTagKey::Comment) =>
if let Value::String(s) = &tag.value {
info.comments = Some(s.clone());
},
Some(StandardTagKey::Genre) =>
if let Value::String(s) = &tag.value {
info.genre = Some(s.clone());
},
Some(StandardTagKey::TrackTitle) =>
if let Value::String(s) = &tag.value {
info.title = Some(s.clone());
},
Some(StandardTagKey::Url | StandardTagKey::UrlSource) => {
if let Value::String(s) = &tag.value {
origin.url = Some(s.clone());
}
},
_ => {},
}
}
for _visual in meta.visuals() {
// FIXME: will require MIME type inspection and Base64 conversion.
}
}
}
/// Transform applied inside [`Compressed`], converting a floating-point PCM
/// input stream into a DCA-framed Opus stream.
///
/// Created and managed by [`Compressed`].
///
/// [`Compressed`]: Compressed
#[derive(Debug)]
pub struct OpusCompressor {
prepend: Option<Cursor<Vec<u8>>>,
encoder: OpusEncoder,
last_frame: Vec<u8>,
stereo_input: bool,
frame_pos: usize,
audio_bytes: AtomicUsize,
}
impl OpusCompressor {
fn new(encoder: OpusEncoder, stereo_input: bool, prepend: Vec<u8>) -> Self {
Self {
prepend: Some(Cursor::new(prepend)),
encoder,
last_frame: Vec::with_capacity(4000),
stereo_input,
frame_pos: 0,
audio_bytes: AtomicUsize::default(),
}
}
}
impl<T> Transform<T> for OpusCompressor
where
T: Read,
{
fn transform_read(&mut self, src: &mut T, buf: &mut [u8]) -> IoResult<TransformPosition> {
if let Some(prepend) = self.prepend.as_mut() {
match prepend.read(buf)? {
0 => {},
n => return Ok(TransformPosition::Read(n)),
}
}
self.prepend = None;
let output_start = mem::size_of::<u16>();
let mut eof = false;
let mut raw_len = 0;
let mut out = None;
let mut sample_buf = [0f32; STEREO_FRAME_SIZE];
let (samples_in_frame, interleaved_count) = if self.stereo_input {
(STEREO_FRAME_SIZE, 2)
} else {
(MONO_FRAME_SIZE, 1)
};
// Purge old frame and read new, if needed.
if self.frame_pos == self.last_frame.len() + output_start || self.last_frame.is_empty() {
self.last_frame.resize(self.last_frame.capacity(), 0);
// We can't use `read_f32_into` because we can't guarantee the buffer will be filled.
// However, we can guarantee that reads will be channel aligned at least!
for el in sample_buf[..samples_in_frame].chunks_mut(interleaved_count) {
match src.read_f32_into::<LittleEndian>(el) {
Ok(()) => {
raw_len += interleaved_count;
},
Err(e) if e.kind() == IoErrorKind::UnexpectedEof => {
eof = true;
break;
},
Err(e) => {
out = Some(Err(e));
break;
},
}
}
if out.is_none() && raw_len > 0 {
loop {
// NOTE: we don't index by raw_len because the last frame can be too small
// to occupy a "whole packet". Zero-padding is the correct behaviour.
match self
.encoder
.encode_float(&sample_buf[..samples_in_frame], &mut self.last_frame[..])
{
Ok(pkt_len) => {
trace!("Next packet to write has {:?}", pkt_len);
self.frame_pos = 0;
self.last_frame.truncate(pkt_len);
break;
},
Err(e) if e.code() == OpusErrorCode::BufferTooSmall => {
// If we need more capacity to encode this frame, then take it.
trace!("Resizing inner buffer (+256).");
self.last_frame.resize(self.last_frame.len() + 256, 0);
},
Err(e) => {
debug!("Read error {:?} {:?} {:?}.", e, out, raw_len);
out = Some(Err(IoError::other(e)));
break;
},
}
}
}
}
if out.is_none() {
// Write from frame we have.
let start = if self.frame_pos < output_start {
(&mut buf[..output_start])
.write_i16::<LittleEndian>(self.last_frame.len() as i16)
.expect(
"Minimum bytes requirement for Opus (2) should mean that an i16 \
may always be written.",
);
self.frame_pos += output_start;
trace!("Wrote frame header: {}.", self.last_frame.len());
output_start
} else {
0
};
let out_pos = self.frame_pos - output_start;
let remaining = self.last_frame.len() - out_pos;
let write_len = remaining.min(buf.len() - start);
buf[start..start + write_len]
.copy_from_slice(&self.last_frame[out_pos..out_pos + write_len]);
self.frame_pos += write_len;
trace!("Appended {} to inner store", write_len);
out = Some(Ok(write_len + start));
}
// NOTE: use of raw_len here preserves true sample length even if
// stream is extended to 20ms boundary.
out.unwrap_or_else(|| Err(IoError::other("Unclear.")))
.map(|compressed_sz| {
self.audio_bytes
.fetch_add(raw_len * mem::size_of::<f32>(), Ordering::Release);
if eof {
TransformPosition::Finished
} else {
TransformPosition::Read(compressed_sz)
}
})
}
}
impl NeedsBytes for OpusCompressor {
fn min_bytes_required(&self) -> usize {
2
}
}
impl Stateful for OpusCompressor {
type State = usize;
fn state(&self) -> Self::State {
self.audio_bytes.load(Ordering::Acquire)
}
}
impl Read for Compressed {
fn read(&mut self, buf: &mut [u8]) -> IoResult<usize> {
self.raw.read(buf)
}
}
impl Seek for Compressed {
fn seek(&mut self, pos: SeekFrom) -> IoResult<u64> {
self.raw.seek(pos)
}
}
impl MediaSource for Compressed {
fn is_seekable(&self) -> bool {
true
}
fn byte_len(&self) -> Option<u64> {
if self.raw.is_finished() {
Some(self.raw.len() as u64)
} else {
None
}
}
}
impl From<Compressed> for Input {
fn from(val: Compressed) -> Input {
let input = Box::new(val);
Input::Live(LiveInput::Raw(AudioStream { input }), None)
}
}

View File

@@ -0,0 +1,140 @@
use super::{compressed::Config, CodecCacheError, ToAudioBytes};
use crate::{
constants::SAMPLE_RATE_RAW,
input::{AudioStream, Input, LiveInput, RawAdapter},
};
use std::io::{Read, Result as IoResult, Seek, SeekFrom};
use streamcatcher::Catcher;
use symphonia_core::{audio::Channels, io::MediaSource};
/// A wrapper around an existing [`Input`] which caches
/// the decoded and converted audio data locally in memory
/// as `f32`-format PCM data.
///
/// The main purpose of this wrapper is to enable seeking on
/// incompatible sources (i.e., ffmpeg output) and to ease resource
/// consumption for commonly reused/shared tracks. [`Compressed`]
/// offers similar functionality with different
/// tradeoffs.
///
/// This is intended for use with small, repeatedly used audio
/// tracks shared between sources, and stores the sound data
/// retrieved in **uncompressed floating point** form to minimise the
/// cost of audio processing when mixing several tracks together.
/// This must be used sparingly: these cost a significant
/// *3 Mbps (375 kiB/s)*, or 131 MiB of RAM for a 6 minute song.
///
/// [`Input`]: crate::input::Input
/// [`Compressed`]: super::Compressed
#[derive(Clone)]
pub struct Decompressed {
/// Inner shared bytestore.
pub raw: Catcher<RawAdapter<ToAudioBytes>>,
}
impl Decompressed {
/// Wrap an existing [`Input`] with an in-memory store, decompressed into `f32` PCM audio.
///
/// [`Input`]: Input
pub async fn new(source: Input) -> Result<Self, CodecCacheError> {
Self::with_config(source, None).await
}
/// Wrap an existing [`Input`] with an in-memory store, decompressed into `f32` PCM audio,
/// with custom configuration for both Symphonia and the backing store.
///
/// [`Input`]: Input
pub async fn with_config(
source: Input,
config: Option<Config>,
) -> Result<Self, CodecCacheError> {
let input = match source {
Input::Lazy(mut r) => {
let created = if r.should_create_async() {
r.create_async().await.map_err(CodecCacheError::from)
} else {
tokio::task::spawn_blocking(move || r.create().map_err(CodecCacheError::from))
.await
.map_err(CodecCacheError::from)
.and_then(|v| v)
};
created.map(LiveInput::Raw)
},
Input::Live(LiveInput::Parsed(_), _) => Err(CodecCacheError::StreamNotAtStart),
Input::Live(a, _rec) => Ok(a),
}?;
let cost_per_sec = super::raw_cost_per_sec(true);
let config = config.unwrap_or_else(|| Config::default_from_cost(cost_per_sec));
let promoted = tokio::task::spawn_blocking(move || {
input.promote(config.codec_registry, config.format_registry)
})
.await??;
// If success, guaranteed to be Parsed
let LiveInput::Parsed(parsed) = promoted else {
unreachable!()
};
let track_info = parsed.decoder.codec_params();
let chan_count = track_info
.channels
.map(Channels::count)
.ok_or(CodecCacheError::UnknownChannelCount)?;
let sample_rate = SAMPLE_RATE_RAW as u32;
let source = RawAdapter::new(
ToAudioBytes::new(parsed, Some(chan_count)),
sample_rate,
chan_count as u32,
);
let raw = config.streamcatcher.build(source)?;
Ok(Self { raw })
}
/// Acquire a new handle to this object, creating a new
/// view of the existing cached data from the beginning.
#[must_use]
pub fn new_handle(&self) -> Self {
Self {
raw: self.raw.new_handle(),
}
}
}
impl Read for Decompressed {
fn read(&mut self, buf: &mut [u8]) -> IoResult<usize> {
self.raw.read(buf)
}
}
impl Seek for Decompressed {
fn seek(&mut self, pos: SeekFrom) -> IoResult<u64> {
self.raw.seek(pos)
}
}
impl MediaSource for Decompressed {
fn is_seekable(&self) -> bool {
true
}
fn byte_len(&self) -> Option<u64> {
if self.raw.is_finished() {
Some(self.raw.len() as u64)
} else {
None
}
}
}
impl From<Decompressed> for Input {
fn from(val: Decompressed) -> Input {
let input = Box::new(val);
Input::Live(LiveInput::Raw(AudioStream { input }), None)
}
}

View File

@@ -0,0 +1,144 @@
use crate::{error::JsonError, input::AudioStreamError};
use opus2::Error as OpusError;
use std::{
error::Error as StdError,
fmt::{Display, Formatter, Result as FmtResult},
};
use streamcatcher::CatcherError;
use symphonia_core::errors::Error as SymphError;
use tokio::task::JoinError;
/// Errors encountered using a [`Memory`] cached source.
///
/// [`Memory`]: super::Memory
#[derive(Debug)]
pub enum Error {
/// The audio stream could not be created.
Create(AudioStreamError),
/// The audio stream failed to be created due to a panic in `spawn_blocking`.
CreatePanicked,
/// Streamcatcher's configuration was illegal, and the cache could not be created.
Streamcatcher(CatcherError),
/// The input stream had already been read (i.e., `Parsed`) and so the whole stream
/// could not be used.
StreamNotAtStart,
}
impl Display for Error {
fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult {
match self {
Self::Create(c) => f.write_fmt(format_args!("failed to create audio stream: {c}")),
Self::CreatePanicked => f.write_str("sync thread panicked while creating stream"),
Self::Streamcatcher(s) =>
f.write_fmt(format_args!("illegal streamcatcher config: {s}")),
Self::StreamNotAtStart =>
f.write_str("stream cannot have been pre-read/parsed, missing headers"),
}
}
}
impl StdError for Error {}
impl From<AudioStreamError> for Error {
fn from(val: AudioStreamError) -> Self {
Self::Create(val)
}
}
impl From<CatcherError> for Error {
fn from(val: CatcherError) -> Self {
Self::Streamcatcher(val)
}
}
impl From<JoinError> for Error {
fn from(_val: JoinError) -> Self {
Self::CreatePanicked
}
}
/// Errors encountered using a [`Compressed`] or [`Decompressed`] cached source.
///
/// [`Compressed`]: super::Compressed
/// [`Decompressed`]: super::Decompressed
#[derive(Debug)]
pub enum CodecCacheError {
/// The audio stream could not be created.
Create(AudioStreamError),
/// Symphonia failed to parse the container or decode the default stream.
Parse(SymphError),
/// The Opus encoder could not be created.
Opus(OpusError),
/// The file's metadata could not be converted to JSON.
MetadataEncoding(JsonError),
/// The input's metadata was too large after conversion to JSON to fit in a DCA file.
MetadataTooLarge,
/// The audio stream failed to be created due to a panic in `spawn_blocking`.
CreatePanicked,
/// The audio stream's channel count could not be determined.
UnknownChannelCount,
/// Streamcatcher's configuration was illegal, and the cache could not be created.
Streamcatcher(CatcherError),
/// The input stream had already been read (i.e., `Parsed`) and so the whole stream
/// could not be used.
StreamNotAtStart,
}
impl Display for CodecCacheError {
fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult {
match self {
Self::Create(c) => f.write_fmt(format_args!("failed to create audio stream: {c}")),
Self::Parse(p) => f.write_fmt(format_args!("failed to parse audio format: {p}")),
Self::Opus(o) => f.write_fmt(format_args!("failed to create Opus encoder: {o}")),
Self::MetadataEncoding(m) => f.write_fmt(format_args!(
"failed to convert track metadata to JSON: {m}"
)),
Self::MetadataTooLarge => f.write_str("track metadata was too large, >= 32kiB"),
Self::CreatePanicked => f.write_str("sync thread panicked while creating stream"),
Self::UnknownChannelCount =>
f.write_str("audio stream's channel count could not be determined"),
Self::Streamcatcher(s) =>
f.write_fmt(format_args!("illegal streamcatcher config: {s}")),
Self::StreamNotAtStart =>
f.write_str("stream cannot have been pre-read/parsed, missing headers"),
}
}
}
impl StdError for CodecCacheError {}
impl From<AudioStreamError> for CodecCacheError {
fn from(val: AudioStreamError) -> Self {
Self::Create(val)
}
}
impl From<CatcherError> for CodecCacheError {
fn from(val: CatcherError) -> Self {
Self::Streamcatcher(val)
}
}
impl From<JoinError> for CodecCacheError {
fn from(_val: JoinError) -> Self {
Self::CreatePanicked
}
}
impl From<JsonError> for CodecCacheError {
fn from(val: JsonError) -> Self {
Self::MetadataEncoding(val)
}
}
impl From<OpusError> for CodecCacheError {
fn from(val: OpusError) -> Self {
Self::Opus(val)
}
}
impl From<SymphError> for CodecCacheError {
fn from(val: SymphError) -> Self {
Self::Parse(val)
}
}

View File

@@ -0,0 +1,40 @@
use std::time::Duration;
use streamcatcher::Config;
/// Expected amount of time that an input should last.
#[derive(Copy, Clone, Debug)]
pub enum LengthHint {
/// Estimate of a source's length in bytes.
Bytes(usize),
/// Estimate of a source's length in time.
///
/// This will be converted to a bytecount at setup.
Time(Duration),
}
impl From<usize> for LengthHint {
fn from(size: usize) -> Self {
LengthHint::Bytes(size)
}
}
impl From<Duration> for LengthHint {
fn from(size: Duration) -> Self {
LengthHint::Time(size)
}
}
/// Modify the given cache configuration to initially allocate
/// enough bytes to store a length of audio at the given bitrate.
pub fn apply_length_hint<H>(config: &mut Config, hint: H, cost_per_sec: usize)
where
H: Into<LengthHint>,
{
config.length_hint = Some(match hint.into() {
LengthHint::Bytes(a) => a,
LengthHint::Time(t) => {
let s = t.as_secs() + u64::from(t.subsec_millis() > 0);
(s as usize) * cost_per_sec
},
});
}

View File

@@ -0,0 +1,111 @@
use super::{default_config, raw_cost_per_sec, Error};
use crate::input::{AudioStream, Input, LiveInput};
use std::io::{Read, Result as IoResult, Seek};
use streamcatcher::{Catcher, Config};
use symphonia_core::io::MediaSource;
/// A wrapper around an existing [`Input`] which caches its data
/// in memory.
///
/// The main purpose of this wrapper is to enable fast seeking on
/// incompatible sources (i.e., HTTP streams) and to ease resource
/// consumption for commonly reused/shared tracks.
///
/// This consumes exactly as many bytes of memory as the input stream contains.
///
/// [`Input`]: Input
#[derive(Clone)]
pub struct Memory {
/// Inner shared bytestore.
pub raw: Catcher<Box<dyn MediaSource>>,
}
impl Memory {
/// Wrap an existing [`Input`] with an in-memory store with the same codec and framing.
///
/// [`Input`]: Input
pub async fn new(source: Input) -> Result<Self, Error> {
Self::with_config(source, None).await
}
/// Wrap an existing [`Input`] with an in-memory store with the same codec and framing.
///
/// `length_hint` may be used to control the size of the initial chunk, preventing
/// needless allocations and copies.
///
/// [`Input`]: Input
pub async fn with_config(source: Input, config: Option<Config>) -> Result<Self, Error> {
let input = match source {
Input::Lazy(mut r) => {
let created = if r.should_create_async() {
r.create_async().await
} else {
tokio::task::spawn_blocking(move || r.create()).await?
};
created.map(|v| v.input).map_err(Error::from)
},
Input::Live(LiveInput::Raw(a), _rec) => Ok(a.input),
Input::Live(LiveInput::Wrapped(a), _rec) =>
Ok(Box::new(a.input) as Box<dyn MediaSource>),
Input::Live(LiveInput::Parsed(_), _) => Err(Error::StreamNotAtStart),
}?;
let cost_per_sec = raw_cost_per_sec(true);
let config = config.unwrap_or_else(|| default_config(cost_per_sec));
// TODO: apply length hint.
// if config.length_hint.is_none() {
// if let Some(dur) = metadata.duration {
// apply_length_hint(&mut config, dur, cost_per_sec);
// }
// }
let raw = config.build(input)?;
Ok(Self { raw })
}
/// Acquire a new handle to this object, creating a new
/// view of the existing cached data from the beginning.
#[must_use]
pub fn new_handle(&self) -> Self {
Self {
raw: self.raw.new_handle(),
}
}
}
impl Read for Memory {
fn read(&mut self, buf: &mut [u8]) -> IoResult<usize> {
self.raw.read(buf)
}
}
impl Seek for Memory {
fn seek(&mut self, pos: std::io::SeekFrom) -> IoResult<u64> {
self.raw.seek(pos)
}
}
impl MediaSource for Memory {
fn is_seekable(&self) -> bool {
true
}
fn byte_len(&self) -> Option<u64> {
if self.raw.is_finished() {
Some(self.raw.len() as u64)
} else {
None
}
}
}
impl From<Memory> for Input {
fn from(val: Memory) -> Input {
let input = Box::new(val);
Input::Live(LiveInput::Raw(AudioStream { input }), None)
}
}

View File

@@ -0,0 +1,49 @@
//! In-memory, shared input sources for reuse between calls, fast seeking, and
//! direct Opus frame passthrough.
mod compressed;
mod decompressed;
mod error;
mod hint;
mod memory;
mod util;
pub(crate) use self::util::*;
pub use self::{compressed::*, decompressed::*, error::*, hint::*, memory::*};
use crate::constants::*;
use crate::input::utils;
use opus2::Bitrate;
use std::{mem, time::Duration};
use streamcatcher::{Config as ScConfig, GrowthStrategy};
/// Estimates the cost, in B/s, of audio data compressed at the given bitrate.
#[must_use]
pub fn compressed_cost_per_sec(bitrate: Bitrate) -> usize {
let framing_cost_per_sec = AUDIO_FRAME_RATE * mem::size_of::<u16>();
let bitrate_raw = match bitrate {
Bitrate::Bits(i) => i,
Bitrate::Auto => 64_000,
Bitrate::Max => 512_000,
} as usize;
(bitrate_raw / 8) + framing_cost_per_sec
}
/// Calculates the cost, in B/s, of raw floating-point audio data.
#[must_use]
pub fn raw_cost_per_sec(stereo: bool) -> usize {
utils::timestamp_to_byte_count(Duration::from_secs(1), stereo)
}
/// Provides the default config used by a cached source.
///
/// This maps to the default configuration in [`streamcatcher`], using
/// a constant chunk size of 5s worth of audio at the given bitrate estimate.
///
/// [`streamcatcher`]: https://docs.rs/streamcatcher/0.1.0/streamcatcher/struct.Config.html
#[must_use]
pub fn default_config(cost_per_sec: usize) -> ScConfig {
ScConfig::new().chunk_size(GrowthStrategy::Constant(5 * cost_per_sec))
}

View File

@@ -0,0 +1,458 @@
use crate::{constants::*, driver::tasks::mixer::mix_logic, input::Parsed};
use byteorder::{LittleEndian, WriteBytesExt};
use rubato::{FftFixedOut, Resampler};
use std::{
io::{ErrorKind as IoErrorKind, Read, Result as IoResult, Seek, Write},
mem,
ops::Range,
};
use symphonia_core::{
audio::{AudioBuffer, AudioBufferRef, Layout, Signal, SignalSpec},
conv::IntoSample,
io::MediaSource,
sample::Sample,
};
const SAMPLE_LEN: usize = mem::size_of::<f32>();
/// Adapter for Symphonia sources into an interleaved f32 bytestream.
///
/// This will output `f32`s in LE byte order, matching the channel count
/// of the input.
pub struct ToAudioBytes {
chan_count: usize,
chan_limit: usize,
parsed: Parsed,
/// Position with parsed's last decoded frame.
inner_pos: Range<usize>,
resample: Option<ResampleState>,
done: bool,
interrupted_samples: Vec<f32>,
interrupted_byte_pos: Range<usize>,
}
struct ResampleState {
/// Used to hold outputs from resampling, *ready to be used*.
resampled_data: Vec<Vec<f32>>,
/// The actual resampler.
resampler: FftFixedOut<f32>,
/// Used to hold inputs to resampler across packet boundaries.
scratch: AudioBuffer<f32>,
/// The range of floats in `resampled_data` which have not yet
/// been read.
resample_pos: Range<usize>,
}
impl ToAudioBytes {
pub fn new(parsed: Parsed, chan_limit: Option<usize>) -> Self {
let track_info = parsed.decoder.codec_params();
let sample_rate = track_info.sample_rate.unwrap_or(SAMPLE_RATE_RAW as u32);
let maybe_layout = track_info.channel_layout;
let maybe_chans = track_info.channels;
let chan_count = if let Some(chans) = maybe_chans {
chans.count()
} else if let Some(layout) = maybe_layout {
match layout {
Layout::Mono => 1,
Layout::Stereo => 2,
Layout::TwoPointOne => 3,
Layout::FivePointOne => 6,
}
} else {
2
};
let chan_limit = chan_limit.unwrap_or(chan_count);
let resample = (sample_rate != SAMPLE_RATE_RAW as u32).then(|| {
let spec = if let Some(chans) = maybe_chans {
SignalSpec::new(SAMPLE_RATE_RAW as u32, chans)
} else if let Some(layout) = maybe_layout {
SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, layout)
} else {
SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, Layout::Stereo)
};
let scratch = AudioBuffer::<f32>::new(MONO_FRAME_SIZE as u64, spec);
// TODO: integ. error handling here.
let resampler = FftFixedOut::new(
sample_rate as usize,
SAMPLE_RATE_RAW,
RESAMPLE_OUTPUT_FRAME_SIZE,
4,
chan_count,
)
.expect("Failed to create resampler.");
let resampled_data = resampler.output_buffer_allocate(true);
ResampleState {
resampled_data,
resampler,
scratch,
resample_pos: 0..0,
}
});
Self {
chan_count,
chan_limit,
parsed,
inner_pos: 0..0,
resample,
done: false,
interrupted_samples: Vec::with_capacity(chan_count),
interrupted_byte_pos: 0..0,
}
}
pub fn num_channels(&self) -> usize {
self.chan_count.min(self.chan_limit)
}
fn is_done(&self) -> bool {
self.done
&& self.inner_pos.is_empty()
&& self
.resample
.as_ref()
.is_none_or(|v| v.scratch.frames() == 0 && v.resample_pos.is_empty())
&& self.interrupted_byte_pos.is_empty()
}
}
impl Read for ToAudioBytes {
fn read(&mut self, mut buf: &mut [u8]) -> IoResult<usize> {
// NOTE: this is disturbingly similar to the mixer code, but different enough that we can't
// just reuse it freely.
let orig_sz = buf.len();
let num_chans = self.num_channels();
while !buf.is_empty() && !self.is_done() {
// Work to clear interrupted channel floats.
while !buf.is_empty() && !self.interrupted_byte_pos.is_empty() {
let index_of_first_f32 = self.interrupted_byte_pos.start / SAMPLE_LEN;
let f32_inner_pos = self.interrupted_byte_pos.start % SAMPLE_LEN;
let f32_bytes_remaining = SAMPLE_LEN - f32_inner_pos;
let to_write = f32_bytes_remaining.min(buf.len());
let bytes = self.interrupted_samples[index_of_first_f32].to_le_bytes();
let written = buf.write(&bytes[f32_inner_pos..][..to_write])?;
self.interrupted_byte_pos.start += written;
}
// Clear out already produced resampled floats.
if let Some(resample) = self.resample.as_mut() {
if !buf.is_empty() && !resample.resample_pos.is_empty() {
let bytes_advanced = write_resample_buffer(
&resample.resampled_data,
buf,
&mut resample.resample_pos,
&mut self.interrupted_samples,
&mut self.interrupted_byte_pos,
num_chans,
);
buf = &mut buf[bytes_advanced..];
}
if !resample.resample_pos.is_empty() {
continue;
}
}
// Now work with new packets.
let source_packet = if !self.inner_pos.is_empty() {
Some(self.parsed.decoder.last_decoded())
} else if let Ok(pkt) = self.parsed.format.next_packet() {
if pkt.track_id() != self.parsed.track_id {
continue;
}
self.parsed
.decoder
.decode(&pkt)
.inspect(|pkt| {
self.inner_pos = 0..pkt.frames();
})
.ok()
} else {
// EOF.
None
};
if source_packet.is_none() {
self.done = true;
if let Some(resample) = self.resample.as_mut() {
if resample.scratch.frames() != 0 {
let data = &mut resample.resampled_data;
let resampler = &mut resample.resampler;
let in_len = resample.scratch.frames();
let to_render = resampler.input_frames_next().saturating_sub(in_len);
if to_render != 0 {
resample.scratch.render_reserved(Some(to_render));
for plane in resample.scratch.planes_mut().planes() {
for val in &mut plane[in_len..] {
*val = 0.0f32;
}
}
}
// Luckily, we make use of the WHOLE input buffer here.
resampler
.process_into_buffer(resample.scratch.planes().planes(), data, None)
.unwrap();
// Calculate true end position using sample rate math
let ratio = (data[0].len() as f32) / (resample.scratch.frames() as f32);
let out_samples = (ratio * (in_len as f32)).round() as usize;
resample.scratch.clear();
resample.resample_pos = 0..out_samples;
}
}
// Now go back and make use of the buffer.
// We have to do this here because we can't make any guarantees about
// the read site having enough space to hold all samples etc.
continue;
}
let source_packet = source_packet.unwrap();
if let Some(resample) = self.resample.as_mut() {
// Do a resample using the newest packet.
let pkt_frames = source_packet.frames();
if pkt_frames == 0 {
continue;
}
let needed_in_frames = resample.resampler.input_frames_next();
let available_frames = self.inner_pos.len();
let force_copy =
resample.scratch.frames() != 0 || needed_in_frames > available_frames;
if (!force_copy) && matches!(source_packet, AudioBufferRef::F32(_)) {
// This is the only case where we can pull off a straight resample...
// I.e., skip scratch.
// NOTE: if let needed as if-let && {bool} is nightly only.
if let AudioBufferRef::F32(s_pkt) = source_packet {
let refs: Vec<&[f32]> = s_pkt
.planes()
.planes()
.iter()
.map(|s| &s[self.inner_pos.start..][..needed_in_frames])
.collect();
self.inner_pos.start += needed_in_frames;
resample
.resampler
.process_into_buffer(&refs, &mut resample.resampled_data, None)
.unwrap();
} else {
unreachable!()
}
} else {
// We either lack enough samples, or have the wrong data format, forcing
// a conversion/copy into scratch.
let old_scratch_len = resample.scratch.frames();
let missing_frames = needed_in_frames - old_scratch_len;
let frames_to_take = available_frames.min(missing_frames);
resample.scratch.render_reserved(Some(frames_to_take));
mix_logic::copy_into_resampler(
&source_packet,
&mut resample.scratch,
self.inner_pos.start,
old_scratch_len,
frames_to_take,
);
self.inner_pos.start += frames_to_take;
if resample.scratch.frames() == needed_in_frames {
resample
.resampler
.process_into_buffer(
resample.scratch.planes().planes(),
&mut resample.resampled_data,
None,
)
.unwrap();
resample.scratch.clear();
} else {
continue;
}
}
resample.resample_pos = 0..resample.resampled_data[0].len();
} else {
// Newest packet may be used straight away: just convert format
// to ensure it's f32.
let bytes_advanced = write_out(
&source_packet,
buf,
&mut self.inner_pos,
&mut self.interrupted_samples,
&mut self.interrupted_byte_pos,
num_chans,
);
buf = &mut buf[bytes_advanced..];
}
}
Ok(orig_sz - buf.len())
}
}
impl Seek for ToAudioBytes {
fn seek(&mut self, _pos: std::io::SeekFrom) -> IoResult<u64> {
Err(IoErrorKind::Unsupported.into())
}
}
impl MediaSource for ToAudioBytes {
fn is_seekable(&self) -> bool {
false
}
fn byte_len(&self) -> Option<u64> {
None
}
}
#[inline]
fn write_out(
source: &AudioBufferRef<'_>,
target: &mut [u8],
source_pos: &mut Range<usize>,
spillover: &mut Vec<f32>,
spill_range: &mut Range<usize>,
num_chans: usize,
) -> usize {
match source {
AudioBufferRef::U8(v) =>
write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans),
AudioBufferRef::U16(v) =>
write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans),
AudioBufferRef::U24(v) =>
write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans),
AudioBufferRef::U32(v) =>
write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans),
AudioBufferRef::S8(v) =>
write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans),
AudioBufferRef::S16(v) =>
write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans),
AudioBufferRef::S24(v) =>
write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans),
AudioBufferRef::S32(v) =>
write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans),
AudioBufferRef::F32(v) =>
write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans),
AudioBufferRef::F64(v) =>
write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans),
}
}
#[inline]
fn write_symph_buffer<S>(
source: &AudioBuffer<S>,
buf: &mut [u8],
source_pos: &mut Range<usize>,
spillover: &mut Vec<f32>,
spill_range: &mut Range<usize>,
num_chans: usize,
) -> usize
where
S: Sample + IntoSample<f32>,
{
let float_space = buf.len() / SAMPLE_LEN;
let interleaved_space = float_space / num_chans;
let non_contiguous_end = (float_space % num_chans) != 0;
let remaining = source_pos.len();
let to_write = remaining.min(interleaved_space);
let need_spill = non_contiguous_end && to_write < remaining;
let samples_used = to_write + usize::from(need_spill);
let last_sample = source_pos.start + to_write;
if need_spill {
spillover.clear();
*spill_range = 0..num_chans * SAMPLE_LEN;
}
for (i, plane) in source.planes().planes()[..num_chans].iter().enumerate() {
for (j, sample) in plane[source_pos.start..][..to_write].iter().enumerate() {
// write this into the correct slot of buf.
let addr = ((j * num_chans) + i) * SAMPLE_LEN;
(&mut buf[addr..][..SAMPLE_LEN])
.write_f32::<LittleEndian>((*sample).into_sample())
.expect("Address known to exist by length checks.");
}
if need_spill {
spillover.push(plane[last_sample].into_sample());
}
}
source_pos.start += samples_used;
to_write * num_chans * SAMPLE_LEN
}
#[inline]
fn write_resample_buffer(
source: &[Vec<f32>],
buf: &mut [u8],
source_pos: &mut Range<usize>,
spillover: &mut Vec<f32>,
spill_range: &mut Range<usize>,
num_chans: usize,
) -> usize {
let float_space = buf.len() / SAMPLE_LEN;
let interleaved_space = float_space / num_chans;
let non_contiguous_end = (float_space % num_chans) != 0;
let remaining = source_pos.len();
let to_write = remaining.min(interleaved_space);
let need_spill = non_contiguous_end && to_write < remaining;
let samples_used = to_write + usize::from(need_spill);
let last_sample = source_pos.start + to_write;
if need_spill {
spillover.clear();
*spill_range = 0..num_chans * SAMPLE_LEN;
}
for (i, plane) in source[..num_chans].iter().enumerate() {
for (j, sample) in plane[source_pos.start..][..to_write].iter().enumerate() {
// write this into the correct slot of buf.
let addr = ((j * num_chans) + i) * SAMPLE_LEN;
(&mut buf[addr..][..SAMPLE_LEN])
.write_f32::<LittleEndian>(*sample)
.expect("Address well-formed according to bounds checks.");
}
if need_spill {
spillover.push(plane[last_sample]);
}
}
source_pos.start += samples_used;
to_write * num_chans * SAMPLE_LEN
}

View File

@@ -0,0 +1,90 @@
use crate::input::{AudioStream, Input, LiveInput};
use std::{
io::{Read, Result as IoResult},
mem,
process::Child,
};
use symphonia_core::io::{MediaSource, ReadOnlySource};
use tokio::runtime::Handle;
use tracing::debug;
/// Handle for a child process which ensures that any subprocesses are properly closed
/// on drop.
///
/// # Warning
/// To allow proper cleanup of child processes, if you create a process chain you must
/// make sure to use `From<Vec<Child>>`. Here, the *last* process in the `Vec` will be
/// used as the audio byte source.
#[derive(Debug)]
pub struct ChildContainer(pub Vec<Child>);
impl Read for ChildContainer {
fn read(&mut self, buffer: &mut [u8]) -> IoResult<usize> {
match self.0.last_mut() {
Some(ref mut child) => child.stdout.as_mut().unwrap().read(buffer),
None => Ok(0),
}
}
}
impl ChildContainer {
/// Create a new [`ChildContainer`] from a child process
#[must_use]
pub fn new(children: Vec<Child>) -> Self {
Self(children)
}
}
impl From<Child> for ChildContainer {
fn from(container: Child) -> Self {
Self(vec![container])
}
}
impl From<Vec<Child>> for ChildContainer {
fn from(container: Vec<Child>) -> Self {
Self(container)
}
}
impl From<ChildContainer> for Input {
fn from(val: ChildContainer) -> Self {
let audio_stream = AudioStream {
input: Box::new(ReadOnlySource::new(val)) as Box<dyn MediaSource>,
};
Input::Live(LiveInput::Raw(audio_stream), None)
}
}
impl Drop for ChildContainer {
fn drop(&mut self) {
let children = mem::take(&mut self.0);
if let Ok(handle) = Handle::try_current() {
handle.spawn_blocking(move || {
cleanup_child_processes(children);
});
} else {
cleanup_child_processes(children);
}
}
}
fn cleanup_child_processes(mut children: Vec<Child>) {
let attempt = if let Some(child) = children.last_mut() {
child.kill()
} else {
return;
};
let attempt = attempt.and_then(|()| {
children
.iter_mut()
.rev()
.try_for_each(|child| child.wait().map(|_| ()))
});
if let Err(e) = attempt {
debug!("Error awaiting child process: {:?}", e);
}
}

View File

@@ -0,0 +1,6 @@
mod async_adapter;
pub mod cached;
mod child;
mod raw_adapter;
pub use self::{async_adapter::*, child::*, raw_adapter::*};

View File

@@ -0,0 +1,113 @@
use crate::input::{AudioStream, Input, LiveInput};
use byteorder::{LittleEndian, WriteBytesExt};
use std::io::{ErrorKind as IoErrorKind, Read, Result as IoResult, Seek, SeekFrom, Write};
use symphonia::core::io::MediaSource;
// format header is a magic string, followed by two LE u32s (sample rate, channel count)
const FMT_HEADER: &[u8; 16] = b"SbirdRaw\0\0\0\0\0\0\0\0";
/// Adapter around a raw, interleaved, `f32` PCM byte stream.
///
/// This may be used to port legacy songbird audio sources to be compatible with
/// the symphonia backend, particularly those with unknown length (making WAV
/// unsuitable).
///
/// The format is described in [`RawReader`].
///
/// [`RawReader`]: crate::input::codecs::RawReader
pub struct RawAdapter<A> {
prepend: [u8; 16],
inner: A,
pos: u64,
}
impl<A: MediaSource> RawAdapter<A> {
/// Wrap an input PCM byte source to be readable by symphonia.
pub fn new(audio_source: A, sample_rate: u32, channel_count: u32) -> Self {
let mut prepend: [u8; 16] = *FMT_HEADER;
let mut write_space = &mut prepend[8..];
write_space
.write_u32::<LittleEndian>(sample_rate)
.expect("Prepend buffer is sized to include enough space for sample rate.");
write_space
.write_u32::<LittleEndian>(channel_count)
.expect("Prepend buffer is sized to include enough space for number of channels.");
Self {
prepend,
inner: audio_source,
pos: 0,
}
}
}
impl<A: MediaSource> Read for RawAdapter<A> {
fn read(&mut self, mut buf: &mut [u8]) -> IoResult<usize> {
let out = if self.pos < self.prepend.len() as u64 {
let upos = self.pos as usize;
let remaining = self.prepend.len() - upos;
let to_write = buf.len().min(remaining);
buf.write(&self.prepend[upos..][..to_write])
} else {
self.inner.read(buf)
};
if let Ok(n) = out {
self.pos += n as u64;
}
out
}
}
impl<A: MediaSource> Seek for RawAdapter<A> {
fn seek(&mut self, pos: SeekFrom) -> IoResult<u64> {
if self.is_seekable() {
let target_pos = match pos {
SeekFrom::Start(p) => p,
SeekFrom::End(_) => return Err(IoErrorKind::Unsupported.into()),
SeekFrom::Current(p) if p.unsigned_abs() > self.pos =>
return Err(IoErrorKind::InvalidInput.into()),
SeekFrom::Current(p) => (self.pos as i64 + p) as u64,
};
let out = if target_pos as usize <= self.prepend.len() {
self.inner.rewind().map(|()| 0)
} else {
self.inner.seek(SeekFrom::Start(target_pos))
};
match out {
Ok(0) => self.pos = target_pos,
Ok(a) => self.pos = a + self.prepend.len() as u64,
_ => {},
}
out.map(|_| self.pos)
} else {
Err(IoErrorKind::Unsupported.into())
}
}
}
impl<A: MediaSource> MediaSource for RawAdapter<A> {
fn is_seekable(&self) -> bool {
self.inner.is_seekable()
}
fn byte_len(&self) -> Option<u64> {
self.inner.byte_len().map(|m| m + self.prepend.len() as u64)
}
}
impl<A: MediaSource + Send + Sync + 'static> From<RawAdapter<A>> for Input {
fn from(val: RawAdapter<A>) -> Self {
let live = LiveInput::Raw(AudioStream {
input: Box::new(val),
});
Input::Live(live, None)
}
}

View File

@@ -0,0 +1,8 @@
/// An unread byte stream for an audio file.
pub struct AudioStream<T: Send> {
/// The wrapped file stream.
///
/// An input stream *must not* have been read into past the start of the
/// audio container's header.
pub input: T,
}

View File

@@ -0,0 +1,53 @@
use serde::{Deserialize, Serialize};
#[derive(Debug, Deserialize, Serialize)]
pub struct DcaMetadata {
pub dca: DcaInfo,
pub opus: Opus,
pub info: Option<Info>,
pub origin: Option<Origin>,
pub extra: Option<serde_json::Value>,
}
#[derive(Debug, Deserialize, Serialize)]
pub struct DcaInfo {
pub version: u64,
pub tool: Tool,
}
#[derive(Debug, Deserialize, Serialize)]
pub struct Tool {
pub name: String,
pub version: String,
pub url: Option<String>,
pub author: Option<String>,
}
#[derive(Debug, Deserialize, Serialize)]
pub struct Opus {
pub mode: String,
pub sample_rate: u32,
pub frame_size: u64,
pub abr: Option<u64>,
pub vbr: bool,
pub channels: u8,
}
#[derive(Debug, Deserialize, Serialize)]
pub struct Info {
pub title: Option<String>,
pub artist: Option<String>,
pub album: Option<String>,
pub genre: Option<String>,
pub cover: Option<String>,
pub comments: Option<String>,
}
#[derive(Debug, Deserialize, Serialize)]
pub struct Origin {
pub source: Option<String>,
pub abr: Option<u64>,
pub channels: Option<u8>,
pub encoding: Option<String>,
pub url: Option<String>,
}

View File

@@ -0,0 +1,338 @@
mod metadata;
pub use self::metadata::*;
use crate::constants::{SAMPLE_RATE, SAMPLE_RATE_RAW};
use std::io::{Seek, SeekFrom};
use symphonia::core::{
codecs::{CodecParameters, CODEC_TYPE_OPUS},
errors::{self as symph_err, Error as SymphError, Result as SymphResult, SeekErrorKind},
formats::prelude::*,
io::{MediaSource, MediaSourceStream, ReadBytes, SeekBuffered},
meta::{Metadata as SymphMetadata, MetadataBuilder, MetadataLog, StandardTagKey, Tag, Value},
probe::{Descriptor, Instantiate, QueryDescriptor},
sample::SampleFormat,
};
impl QueryDescriptor for DcaReader {
fn query() -> &'static [Descriptor] {
&[symphonia_core::support_format!(
"dca",
"DCA[0/1] Opus Wrapper",
&["dca"],
&[],
&[b"DCA1"]
)]
}
fn score(_context: &[u8]) -> u8 {
255
}
}
struct SeekAccel {
frame_offsets: Vec<(TimeStamp, u64)>,
seek_index_fill_rate: u16,
next_ts: TimeStamp,
}
impl SeekAccel {
fn new(options: FormatOptions, first_frame_byte_pos: u64) -> Self {
let per_s = options.seek_index_fill_rate;
let next_ts = (per_s as u64) * (SAMPLE_RATE_RAW as u64);
Self {
frame_offsets: vec![(0, first_frame_byte_pos)],
seek_index_fill_rate: per_s,
next_ts,
}
}
fn update(&mut self, ts: TimeStamp, pos: u64) {
if ts >= self.next_ts {
self.next_ts += (self.seek_index_fill_rate as u64) * (SAMPLE_RATE_RAW as u64);
self.frame_offsets.push((ts, pos));
}
}
fn get_seek_pos(&self, ts: TimeStamp) -> (TimeStamp, u64) {
let index = self.frame_offsets.partition_point(|&(o_ts, _)| o_ts <= ts) - 1;
self.frame_offsets[index]
}
}
/// [DCA\[0/1\]](https://github.com/bwmarrin/dca) Format reader for Symphonia.
pub struct DcaReader {
source: MediaSourceStream,
track: Option<Track>,
metas: MetadataLog,
seek_accel: SeekAccel,
curr_ts: TimeStamp,
max_ts: Option<TimeStamp>,
held_packet: Option<Packet>,
}
impl FormatReader for DcaReader {
fn try_new(mut source: MediaSourceStream, options: &FormatOptions) -> SymphResult<Self> {
// Read in the magic number to verify it's a DCA file.
let magic = source.read_quad_bytes()?;
// FIXME: make use of the new options.enable_gapless to apply the opus coder delay.
let read_meta = match &magic {
b"DCA1" => true,
_ if &magic[..3] == b"DCA" => {
return symph_err::unsupported_error("unsupported DCA version");
},
_ => {
source.seek_buffered_rel(-4);
false
},
};
let mut codec_params = CodecParameters::new();
codec_params
.for_codec(CODEC_TYPE_OPUS)
.with_max_frames_per_packet(1)
.with_sample_rate(SAMPLE_RATE_RAW as u32)
.with_time_base(TimeBase::new(1, SAMPLE_RATE_RAW as u32))
.with_sample_format(SampleFormat::F32);
let mut metas = MetadataLog::default();
if read_meta {
let size = source.read_u32()?;
// Sanity check
if (size as i32) < 2 {
return symph_err::decode_error("missing DCA1 metadata block");
}
let raw_json = source.read_boxed_slice_exact(size as usize)?;
let metadata: DcaMetadata = serde_json::from_slice::<DcaMetadata>(&raw_json)
.map_err(|_| SymphError::DecodeError("malformed DCA1 metadata block"))?;
let mut revision = MetadataBuilder::new();
if let Some(info) = metadata.info {
if let Some(t) = info.title {
revision.add_tag(Tag::new(
Some(StandardTagKey::TrackTitle),
"title",
Value::String(t),
));
}
if let Some(t) = info.album {
revision.add_tag(Tag::new(
Some(StandardTagKey::Album),
"album",
Value::String(t),
));
}
if let Some(t) = info.artist {
revision.add_tag(Tag::new(
Some(StandardTagKey::Artist),
"artist",
Value::String(t),
));
}
if let Some(t) = info.genre {
revision.add_tag(Tag::new(
Some(StandardTagKey::Genre),
"genre",
Value::String(t),
));
}
if let Some(t) = info.comments {
revision.add_tag(Tag::new(
Some(StandardTagKey::Comment),
"comments",
Value::String(t),
));
}
if let Some(_t) = info.cover {
// TODO: Add visual, figure out MIME types.
}
}
if let Some(origin) = metadata.origin {
if let Some(t) = origin.url {
revision.add_tag(Tag::new(Some(StandardTagKey::Url), "url", Value::String(t)));
}
}
metas.push(revision.metadata());
}
let bytes_read = source.pos();
Ok(Self {
source,
track: Some(Track {
id: 0,
language: None,
codec_params,
}),
metas,
seek_accel: SeekAccel::new(*options, bytes_read),
curr_ts: 0,
max_ts: None,
held_packet: None,
})
}
fn cues(&self) -> &[Cue] {
// No cues in DCA...
&[]
}
fn metadata(&mut self) -> SymphMetadata<'_> {
self.metas.metadata()
}
fn seek(&mut self, _mode: SeekMode, to: SeekTo) -> SymphResult<SeekedTo> {
let can_backseek = self.source.is_seekable();
let Some(track) = &self.track else {
return symph_err::seek_error(SeekErrorKind::Unseekable);
};
let rate = track.codec_params.sample_rate;
let ts = match to {
SeekTo::Time { time, .. } =>
if let Some(rate) = rate {
TimeBase::new(1, rate).calc_timestamp(time)
} else {
return symph_err::seek_error(SeekErrorKind::Unseekable);
},
SeekTo::TimeStamp { ts, .. } => ts,
};
if let Some(max_ts) = self.max_ts {
if ts > max_ts {
return symph_err::seek_error(SeekErrorKind::OutOfRange);
}
}
let backseek_needed = self.curr_ts > ts;
if backseek_needed && !can_backseek {
return symph_err::seek_error(SeekErrorKind::ForwardOnly);
}
let (accel_seek_ts, accel_seek_pos) = self.seek_accel.get_seek_pos(ts);
if backseek_needed || accel_seek_pos > self.source.pos() {
self.source.seek(SeekFrom::Start(accel_seek_pos))?;
self.curr_ts = accel_seek_ts;
}
while let Ok(pkt) = self.next_packet() {
let pts = pkt.ts;
let dur = pkt.dur;
let track_id = pkt.track_id();
if (pts..pts + dur).contains(&ts) {
self.held_packet = Some(pkt);
return Ok(SeekedTo {
track_id,
required_ts: ts,
actual_ts: pts,
});
}
}
symph_err::seek_error(SeekErrorKind::OutOfRange)
}
fn tracks(&self) -> &[Track] {
// DCA tracks can hold only one track by design.
// Of course, a zero-length file is technically allowed,
// in which case no track.
self.track.as_slice()
}
fn default_track(&self) -> Option<&Track> {
self.track.as_ref()
}
fn next_packet(&mut self) -> SymphResult<Packet> {
if let Some(pkt) = self.held_packet.take() {
return Ok(pkt);
}
let frame_pos = self.source.pos();
let p_len = match self.source.read_u16() {
Ok(len) => len as i16,
Err(eof) => {
self.max_ts = Some(self.curr_ts);
return Err(eof.into());
},
};
if p_len < 0 {
return symph_err::decode_error("DCA frame header had a negative length.");
}
let buf = self.source.read_boxed_slice_exact(p_len as usize)?;
if buf.is_empty() || buf.len() > i32::MAX as usize {
return symph_err::decode_error(
"Packet was not a valid Opus packet: too large for opus2.",
);
}
let sample_ct = opus2::packet::get_nb_samples(&buf, SAMPLE_RATE).or_else(|_| {
symph_err::decode_error(
"Packet was not a valid Opus packet: couldn't read sample count.",
)
})? as u64;
let out = Packet::new_from_boxed_slice(0, self.curr_ts, sample_ct, buf);
self.seek_accel.update(self.curr_ts, frame_pos);
self.curr_ts += sample_ct;
Ok(out)
}
fn into_inner(self: Box<Self>) -> MediaSourceStream {
self.source
}
}
#[cfg(test)]
mod tests {
use crate::input::input_tests::*;
use crate::{constants::test_data::FILE_DCA_TARGET, input::File};
// NOTE: this covers youtube audio in a non-copyright-violating way, since
// those depend on an HttpRequest internally anyhow.
#[tokio::test]
#[ntest::timeout(10_000)]
async fn dca_track_plays() {
track_plays_passthrough(|| File::new(FILE_DCA_TARGET)).await;
}
#[tokio::test]
#[ntest::timeout(10_000)]
async fn dca_forward_seek_correct() {
forward_seek_correct(|| File::new(FILE_DCA_TARGET)).await;
}
#[tokio::test]
#[ntest::timeout(10_000)]
async fn dca_backward_seek_correct() {
backward_seek_correct(|| File::new(FILE_DCA_TARGET)).await;
}
#[tokio::test]
#[ntest::timeout(10_000)]
async fn opus_passthrough_when_other_tracks_paused() {
track_plays_passthrough_when_is_only_active(|| File::new(FILE_DCA_TARGET)).await;
}
}

View File

@@ -0,0 +1,36 @@
//! Codec registries extending Symphonia's probe and registry formats with Opus and DCA support.
pub(crate) mod dca;
mod opus;
mod raw;
use std::sync::OnceLock;
pub use self::{dca::DcaReader, opus::OpusDecoder, raw::*};
use symphonia::{
core::{codecs::CodecRegistry, probe::Probe},
default::*,
};
/// Default Symphonia [`CodecRegistry`], including the (opus2-backed) Opus codec.
pub fn get_codec_registry() -> &'static CodecRegistry {
static CODEC_REGISTRY: OnceLock<CodecRegistry> = OnceLock::new();
CODEC_REGISTRY.get_or_init(|| {
let mut registry = CodecRegistry::new();
register_enabled_codecs(&mut registry);
registry.register_all::<OpusDecoder>();
registry
})
}
/// Default Symphonia Probe, including DCA format support.
pub fn get_probe() -> &'static Probe {
static PROBE: OnceLock<Probe> = OnceLock::new();
PROBE.get_or_init(|| {
let mut probe = Probe::default();
probe.register_all::<DcaReader>();
probe.register_all::<RawReader>();
register_enabled_formats(&mut probe);
probe
})
}

View File

@@ -0,0 +1,160 @@
use crate::constants::*;
use opus2::{Channels, Decoder as Opus2Decoder, ErrorCode};
use symphonia_core::{
audio::{AsAudioBufferRef, AudioBuffer, AudioBufferRef, Layout, Signal, SignalSpec},
codecs::{
CodecDescriptor,
CodecParameters,
Decoder,
DecoderOptions,
FinalizeResult,
CODEC_TYPE_OPUS,
},
errors::{decode_error, Result as SymphResult},
formats::Packet,
};
/// Opus decoder for symphonia, based on libopus v1.5 (via [`opus2`]).
pub struct OpusDecoder {
inner: Opus2Decoder,
params: CodecParameters,
buf: AudioBuffer<f32>,
rawbuf: Vec<f32>,
}
/// # SAFETY
/// The underlying Opus decoder (currently) requires only a `&self` parameter
/// to decode given packets, which is likely a mistaken decision.
///
/// This struct makes stronger assumptions and only touches FFI decoder state with a
/// `&mut self`, preventing data races via `&OpusDecoder` as required by `impl Sync`.
/// No access to other internal state relies on unsafety or crosses FFI.
unsafe impl Sync for OpusDecoder {}
impl OpusDecoder {
fn decode_inner(&mut self, packet: &Packet) -> SymphResult<()> {
let s_ct = loop {
if packet.buf().len() > i32::MAX as usize {
return decode_error("Opus packet was too large (greater than i32::MAX bytes).");
}
match self
.inner
.decode_float(packet.buf(), &mut self.rawbuf, false)
{
Ok(v) => break v,
Err(e) if e.code() == ErrorCode::BufferTooSmall => {
// double the buffer size
// correct behav would be to mirror the decoder logic in the udp_rx set.
let new_size = (self.rawbuf.len() * 2).min(i32::MAX as usize);
if new_size == self.rawbuf.len() {
return decode_error("Opus frame too big: cannot expand opus frame decode buffer any further.");
}
self.rawbuf.resize(new_size, 0.0);
self.buf = AudioBuffer::new(
self.rawbuf.len() as u64 / 2,
SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, Layout::Stereo),
);
},
Err(e) => {
tracing::error!("Opus decode error: {:?}", e);
return decode_error("Opus decode error: see 'tracing' logs.");
},
}
};
self.buf.clear();
self.buf.render_reserved(Some(s_ct));
// Forcibly assuming stereo, for now.
for ch in 0..2 {
let iter = self.rawbuf.chunks_exact(2).map(|chunk| chunk[ch]);
for (tgt, src) in self.buf.chan_mut(ch).iter_mut().zip(iter) {
*tgt = src;
}
}
Ok(())
}
}
impl Decoder for OpusDecoder {
fn try_new(params: &CodecParameters, _options: &DecoderOptions) -> SymphResult<Self> {
let inner = Opus2Decoder::new(SAMPLE_RATE, Channels::Stereo).unwrap();
let mut params = params.clone();
params.with_sample_rate(SAMPLE_RATE_RAW as u32);
Ok(Self {
inner,
params,
buf: AudioBuffer::new(
MONO_FRAME_SIZE as u64,
SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, Layout::Stereo),
),
rawbuf: vec![0.0f32; STEREO_FRAME_SIZE],
})
}
fn supported_codecs() -> &'static [CodecDescriptor] {
&[symphonia_core::support_codec!(
CODEC_TYPE_OPUS,
"opus",
"libopus (1.5+, opus2)"
)]
}
fn codec_params(&self) -> &CodecParameters {
&self.params
}
fn decode(&mut self, packet: &Packet) -> SymphResult<AudioBufferRef<'_>> {
if let Err(e) = self.decode_inner(packet) {
self.buf.clear();
Err(e)
} else {
Ok(self.buf.as_audio_buffer_ref())
}
}
fn reset(&mut self) {
_ = self.inner.reset_state();
}
fn finalize(&mut self) -> FinalizeResult {
FinalizeResult::default()
}
fn last_decoded(&self) -> AudioBufferRef<'_> {
self.buf.as_audio_buffer_ref()
}
}
#[cfg(test)]
mod tests {
use crate::{
constants::test_data::FILE_WEBM_TARGET,
input::{input_tests::*, File},
};
// NOTE: this covers youtube audio in a non-copyright-violating way, since
// those depend on an HttpRequest internally anyhow.
#[tokio::test]
#[ntest::timeout(10_000)]
async fn webm_track_plays() {
track_plays_passthrough(|| File::new(FILE_WEBM_TARGET)).await;
}
#[tokio::test]
#[ntest::timeout(10_000)]
async fn webm_forward_seek_correct() {
forward_seek_correct(|| File::new(FILE_WEBM_TARGET)).await;
}
#[tokio::test]
#[ntest::timeout(10_000)]
async fn webm_backward_seek_correct() {
backward_seek_correct(|| File::new(FILE_WEBM_TARGET)).await;
}
}

View File

@@ -0,0 +1,181 @@
use std::io::{Seek, SeekFrom};
use symphonia::core::{
audio::Channels,
codecs::{CodecParameters, CODEC_TYPE_PCM_F32LE},
errors::{self as symph_err, Result as SymphResult, SeekErrorKind},
formats::prelude::*,
io::{MediaSource, MediaSourceStream, ReadBytes, SeekBuffered},
meta::{Metadata as SymphMetadata, MetadataLog},
probe::{Descriptor, Instantiate, QueryDescriptor},
};
impl QueryDescriptor for RawReader {
fn query() -> &'static [Descriptor] {
&[symphonia_core::support_format!(
"raw",
"Raw arbitrary-length f32 audio container.",
&["rawf32"],
&[],
&[b"SbirdRaw"]
)]
}
fn score(_context: &[u8]) -> u8 {
255
}
}
/// Symphonia support for a simple container for raw f32-PCM data of unknown duration.
///
/// Contained files have a simple header:
/// * the 8-byte signature `b"SbirdRaw"`,
/// * the sample rate, as a little-endian `u32`,
/// * the channel count, as a little-endian `u32`.
///
/// The remainder of the file is interleaved little-endian `f32` samples.
pub struct RawReader {
source: MediaSourceStream,
track: Track,
meta: MetadataLog,
curr_ts: TimeStamp,
max_ts: Option<TimeStamp>,
}
impl FormatReader for RawReader {
fn try_new(mut source: MediaSourceStream, _options: &FormatOptions) -> SymphResult<Self> {
let mut magic = [0u8; 8];
ReadBytes::read_buf_exact(&mut source, &mut magic[..])?;
if &magic != b"SbirdRaw" {
source.seek_buffered_rel(-(magic.len() as isize));
return symph_err::decode_error("rawf32: illegal magic byte sequence.");
}
let sample_rate = source.read_u32()?;
let n_chans = source.read_u32()?;
let chans = match n_chans {
1 => Channels::FRONT_LEFT,
2 => Channels::FRONT_LEFT | Channels::FRONT_RIGHT,
_ =>
return symph_err::decode_error(
"rawf32: channel layout is not stereo or mono for fmt_pcm",
),
};
let mut codec_params = CodecParameters::new();
codec_params
.for_codec(CODEC_TYPE_PCM_F32LE)
.with_bits_per_coded_sample((std::mem::size_of::<f32>() as u32) * 8)
.with_bits_per_sample((std::mem::size_of::<f32>() as u32) * 8)
.with_sample_rate(sample_rate)
.with_time_base(TimeBase::new(1, sample_rate))
.with_sample_format(symphonia_core::sample::SampleFormat::F32)
.with_max_frames_per_packet(sample_rate as u64 / 50)
.with_channels(chans);
Ok(Self {
source,
track: Track {
id: 0,
language: None,
codec_params,
},
meta: MetadataLog::default(),
curr_ts: 0,
max_ts: None,
})
}
fn cues(&self) -> &[Cue] {
&[]
}
fn metadata(&mut self) -> SymphMetadata<'_> {
self.meta.metadata()
}
fn seek(&mut self, _mode: SeekMode, to: SeekTo) -> SymphResult<SeekedTo> {
let can_backseek = self.source.is_seekable();
let track = &self.track;
let rate = track.codec_params.sample_rate;
let ts = match to {
SeekTo::Time { time, .. } =>
if let Some(rate) = rate {
TimeBase::new(1, rate).calc_timestamp(time)
} else {
return symph_err::seek_error(SeekErrorKind::Unseekable);
},
SeekTo::TimeStamp { ts, .. } => ts,
};
if let Some(max_ts) = self.max_ts {
if ts > max_ts {
return symph_err::seek_error(SeekErrorKind::OutOfRange);
}
}
let backseek_needed = self.curr_ts > ts;
if backseek_needed && !can_backseek {
return symph_err::seek_error(SeekErrorKind::ForwardOnly);
}
let chan_count = track
.codec_params
.channels
.expect("Channel count is built into format.")
.count() as u64;
let seek_pos = 16 + (std::mem::size_of::<f32>() as u64) * (ts * chan_count);
self.source.seek(SeekFrom::Start(seek_pos))?;
self.curr_ts = ts;
Ok(SeekedTo {
track_id: track.id,
required_ts: ts,
actual_ts: ts,
})
}
fn tracks(&self) -> &[Track] {
std::slice::from_ref(&self.track)
}
fn default_track(&self) -> Option<&Track> {
Some(&self.track)
}
fn next_packet(&mut self) -> SymphResult<Packet> {
let track = &self.track;
let rate = track
.codec_params
.sample_rate
.expect("Sample rate is built into format.") as usize;
let chan_count = track
.codec_params
.channels
.expect("Channel count is built into format.")
.count();
let sample_unit = std::mem::size_of::<f32>() * chan_count;
// Aim for 20ms (50Hz).
let buf = self.source.read_boxed_slice((rate / 50) * sample_unit)?;
let sample_ct = (buf.len() / sample_unit) as u64;
let out = Packet::new_from_boxed_slice(0, self.curr_ts, sample_ct, buf);
self.curr_ts += sample_ct;
Ok(out)
}
fn into_inner(self: Box<Self>) -> MediaSourceStream {
self.source
}
}

View File

@@ -0,0 +1,40 @@
use super::{AudioStream, AudioStreamError, AuxMetadata};
use symphonia_core::io::MediaSource;
/// Data and behaviour required to instantiate a lazy audio source.
#[async_trait::async_trait]
pub trait Compose: Send {
/// Create a source synchronously.
///
/// If [`should_create_async`] returns `false`, this method will chosen at runtime.
///
/// [`should_create_async`]: Self::should_create_async
fn create(&mut self) -> Result<AudioStream<Box<dyn MediaSource>>, AudioStreamError>;
/// Create a source asynchronously.
///
/// If [`should_create_async`] returns `true`, this method will chosen at runtime.
///
/// [`should_create_async`]: Self::should_create_async
async fn create_async(&mut self)
-> Result<AudioStream<Box<dyn MediaSource>>, AudioStreamError>;
/// Determines whether this source will be instantiated using [`create`] or [`create_async`].
///
/// Songbird will create the audio stream using either a dynamically sized thread pool,
/// or a task on the async runtime it was spawned in respectively. Users do not need to
/// support both these methods.
///
/// [`create_async`]: Self::create_async
/// [`create`]: Self::create
fn should_create_async(&self) -> bool;
/// Requests auxiliary metadata which can be accessed without parsing the file.
///
/// This method will never be called by songbird but allows, for instance, access to metadata
/// which might only be visible to a web crawler e.g., uploader or source URL.
async fn aux_metadata(&mut self) -> Result<AuxMetadata, AudioStreamError> {
Err(AudioStreamError::Unsupported)
}
}

168
app/vendor/songbird/src/input/error.rs vendored Normal file
View File

@@ -0,0 +1,168 @@
use std::{error::Error, fmt::Display, time::Duration};
use symphonia_core::errors::Error as SymphError;
/// Errors encountered when creating an [`AudioStream`] or requesting metadata
/// from a [`Compose`].
///
/// [`AudioStream`]: super::AudioStream
/// [`Compose`]: super::Compose
#[non_exhaustive]
#[derive(Debug)]
pub enum AudioStreamError {
/// The operation failed, and should be retried after a given time.
///
/// Create operations invoked by the driver will retry on the first tick
/// after this time has passed.
RetryIn(Duration),
/// The operation failed, and should not be retried.
Fail(Box<dyn Error + Send + Sync>),
/// The operation was not supported, and will never succeed.
Unsupported,
}
impl Display for AudioStreamError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("failed to create audio: ")?;
match self {
Self::RetryIn(t) => f.write_fmt(format_args!("retry in {:.2}s", t.as_secs_f32())),
Self::Fail(why) => f.write_fmt(format_args!("{why}")),
Self::Unsupported => f.write_str("operation was not supported"),
}
}
}
impl Error for AudioStreamError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
None
}
}
/// Errors encountered when readying or pre-processing an [`Input`].
///
/// [`Input`]: super::Input
#[non_exhaustive]
#[derive(Debug)]
pub enum MakePlayableError {
/// Failed to create a [`LiveInput`] from the lazy [`Compose`].
///
/// [`LiveInput`]: super::LiveInput
/// [`Compose`]: super::Compose
Create(AudioStreamError),
/// Failed to read headers, codecs, or a valid stream from a [`LiveInput`].
///
/// [`LiveInput`]: super::LiveInput
Parse(SymphError),
/// A blocking thread panicked or failed to return a parsed input.
Panicked,
}
impl Display for MakePlayableError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("failed to make track playable: ")?;
match self {
Self::Create(c) => {
f.write_str("input creation [")?;
f.write_fmt(format_args!("{}", &c))?;
f.write_str("]")
},
Self::Parse(p) => {
f.write_str("parsing formats/codecs [")?;
f.write_fmt(format_args!("{}", &p))?;
f.write_str("]")
},
Self::Panicked => f.write_str("panic during blocking I/O in parse"),
}
}
}
impl Error for MakePlayableError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
None
}
}
impl From<AudioStreamError> for MakePlayableError {
fn from(val: AudioStreamError) -> Self {
Self::Create(val)
}
}
impl From<SymphError> for MakePlayableError {
fn from(val: SymphError) -> Self {
Self::Parse(val)
}
}
/// Errors encountered when trying to access in-stream [`Metadata`] for an [`Input`].
///
/// Both cases can be solved by using [`Input::make_playable`] or [`LiveInput::promote`].
///
/// [`Input`]: super::Input
/// [`Metadata`]: super::Metadata
/// [`Input::make_playable`]: super::Input::make_playable
/// [`LiveInput::promote`]: super::LiveInput::promote
#[non_exhaustive]
#[derive(Debug)]
pub enum MetadataError {
/// This input is currently lazily initialised, and must be made live.
NotLive,
/// This input is ready, but has not had its headers parsed.
NotParsed,
}
impl Display for MetadataError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("failed to get metadata: ")?;
match self {
Self::NotLive => f.write_str("the input is not live, and hasn't been parsed"),
Self::NotParsed => f.write_str("the input is live but hasn't been parsed"),
}
}
}
impl Error for MetadataError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
None
}
}
/// Errors encountered when trying to access out-of-band [`AuxMetadata`] for an [`Input`]
/// or [`Compose`].
///
/// [`Input`]: super::Input
/// [`AuxMetadata`]: super::AuxMetadata
/// [`Compose`]: super::Compose
#[non_exhaustive]
#[derive(Debug)]
pub enum AuxMetadataError {
/// This input has no lazy [`Compose`] initialiser, which is needed to
/// retrieve [`AuxMetadata`].
///
/// [`Compose`]: super::Compose
/// [`AuxMetadata`]: super::AuxMetadata
NoCompose,
/// There was an error when trying to access auxiliary metadata.
Retrieve(AudioStreamError),
}
impl Display for AuxMetadataError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("failed to get aux_metadata: ")?;
match self {
Self::NoCompose => f.write_str("the input has no Compose object"),
Self::Retrieve(e) => f.write_fmt(format_args!("aux_metadata error from Compose: {e}")),
}
}
}
impl Error for AuxMetadataError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
None
}
}
impl From<AudioStreamError> for AuxMetadataError {
fn from(val: AudioStreamError) -> Self {
Self::Retrieve(val)
}
}

View File

@@ -0,0 +1,149 @@
#![allow(missing_docs)]
use crate::{
driver::Driver,
tracks::{PlayMode, ReadyState, Track},
Config,
};
use std::time::Duration;
pub async fn track_plays_passthrough<T, F>(make_track: F)
where
T: Into<Track>,
F: FnOnce() -> T,
{
track_plays_base(make_track, true, None).await;
}
pub async fn track_plays_passthrough_when_is_only_active<T, F>(make_track: F)
where
T: Into<Track>,
F: FnOnce() -> T,
{
track_plays_base(
make_track,
true,
Some(include_bytes!("../../resources/loop.wav")),
)
.await;
}
pub async fn track_plays_mixed<T, F>(make_track: F)
where
T: Into<Track>,
F: FnOnce() -> T,
{
track_plays_base(make_track, false, None).await;
}
pub async fn track_plays_base<T, F>(
make_track: F,
passthrough: bool,
dummy_track: Option<&'static [u8]>,
) where
T: Into<Track>,
F: FnOnce() -> T,
{
let (t_handle, config) = Config::test_cfg(true);
let mut driver = Driver::new(config.clone());
// Used to ensure that paused tracks won't prevent passthrough from happening
// i.e., most queue users :)
if let Some(audio_data) = dummy_track {
driver.play(Track::from(audio_data).pause());
}
let file = make_track();
// Get input in place, playing. Wait for IO to ready.
t_handle.ready_track(&driver.play(file.into()), None).await;
t_handle.tick(1);
// post-conditions:
// 1) track produces a packet.
// 2) that packet is passthrough/mixed when we expect them to.
let pkt = t_handle.recv_async().await;
let pkt = pkt.raw().unwrap();
if passthrough {
assert!(pkt.is_passthrough());
} else {
assert!(pkt.is_mixed_with_nonzero_signal());
}
}
pub async fn forward_seek_correct<T, F>(make_track: F)
where
T: Into<Track>,
F: FnOnce() -> T,
{
let (t_handle, config) = Config::test_cfg(true);
let mut driver = Driver::new(config.clone());
let file = make_track();
let handle = driver.play(file.into());
// Get input in place, playing. Wait for IO to ready.
t_handle.ready_track(&handle, None).await;
let target_time = Duration::from_secs(30);
assert!(!handle.seek(target_time).is_hung_up());
t_handle.ready_track(&handle, None).await;
// post-conditions:
// 1) track is readied
// 2) track's position is approx 30s
// 3) track's play time is considerably less (O(5s))
let state = handle.get_info();
t_handle.spawn_ticker();
let state = state.await.expect("Should have received valid state.");
assert_eq!(state.ready, ReadyState::Playable);
assert_eq!(state.playing, PlayMode::Play);
assert!(state.play_time < Duration::from_secs(5));
assert!(
state.position < target_time + Duration::from_millis(100)
&& state.position > target_time - Duration::from_millis(100)
);
}
pub async fn backward_seek_correct<T, F>(make_track: F)
where
T: Into<Track>,
F: FnOnce() -> T,
{
let (t_handle, config) = Config::test_cfg(true);
let mut driver = Driver::new(config.clone());
let file = make_track();
let handle = driver.play(file.into());
// Get input in place, playing. Wait for IO to ready.
t_handle.ready_track(&handle, None).await;
// Accelerated playout -- 4 seconds worth.
let n_secs = 4;
let n_ticks = 50 * n_secs;
t_handle.skip(n_ticks).await;
let target_time = Duration::from_secs(1);
assert!(!handle.seek(target_time).is_hung_up());
t_handle.ready_track(&handle, None).await;
// post-conditions:
// 1) track is readied
// 2) track's position is approx 1s
// 3) track's play time is preserved (About 4s)
let state = handle.get_info();
t_handle.spawn_ticker();
let state = state.await.expect("Should have received valid state.");
assert_eq!(state.ready, ReadyState::Playable);
assert_eq!(state.playing, PlayMode::Play);
assert!(state.play_time >= Duration::from_secs(n_secs));
assert!(
state.position < target_time + Duration::from_millis(100)
&& state.position > target_time - Duration::from_millis(100)
);
}

View File

@@ -0,0 +1,164 @@
use super::{AudioStream, Metadata, MetadataError, Parsed};
use symphonia_core::{
codecs::{CodecRegistry, DecoderOptions},
errors::Error as SymphError,
formats::FormatOptions,
io::{MediaSource, MediaSourceStream, MediaSourceStreamOptions},
meta::MetadataOptions,
probe::{Hint, Probe},
};
/// An initialised audio source.
///
/// This type's variants reflect files at different stages of readiness for use by
/// symphonia. [`Parsed`] file streams are ready for playback.
///
/// [`Parsed`]: Self::Parsed
pub enum LiveInput {
/// An unread, raw file stream.
Raw(AudioStream<Box<dyn MediaSource>>),
/// An unread file which has been wrapped with a large read-ahead buffer.
Wrapped(AudioStream<MediaSourceStream>),
/// An audio file which has had its headers parsed and decoder state built.
Parsed(Parsed),
}
impl LiveInput {
/// Converts this audio source into a [`Parsed`] object using the supplied format and codec
/// registries.
///
/// Where applicable, this will convert [`Raw`] -> [`Wrapped`] -> [`Parsed`], and will
/// play the default track (or the first encountered track if this is not available) if a
/// container holds multiple audio streams.
///
/// *This is a blocking operation. Symphonia uses standard library I/O (e.g., [`Read`], [`Seek`]).
/// If you wish to use this from an async task, you must do so within `spawn_blocking`.*
///
/// [`Parsed`]: Self::Parsed
/// [`Raw`]: Self::Raw
/// [`Wrapped`]: Self::Wrapped
/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html
/// [`Seek`]: https://doc.rust-lang.org/std/io/trait.Seek.html
pub fn promote(self, codecs: &CodecRegistry, probe: &Probe) -> Result<Self, SymphError> {
let mut out = self;
if let LiveInput::Raw(r) = out {
// TODO: allow passing in of MSS options?
let mss = MediaSourceStream::new(r.input, MediaSourceStreamOptions::default());
out = LiveInput::Wrapped(AudioStream { input: mss });
}
if let LiveInput::Wrapped(w) = out {
let input = w.input;
let supports_backseek = input.is_seekable();
let probe_data = probe.format(
// Hint is ignored
&Hint::new(),
input,
&FormatOptions::default(),
&MetadataOptions::default(),
)?;
let format = probe_data.format;
let meta = probe_data.metadata;
// if default track exists, try to make a decoder
// if that fails, linear scan and take first that succeeds
let decoder = format
.default_track()
.and_then(|track| {
codecs
.make(&track.codec_params, &DecoderOptions::default())
.ok()
.map(|d| (d, track.id))
})
.or_else(|| {
format.tracks().iter().find_map(|track| {
codecs
.make(&track.codec_params, &DecoderOptions::default())
.ok()
.map(|d| (d, track.id))
})
});
// No tracks is a playout error, a bad default track is also possible.
// These are probably malformed? We could go best-effort, and fall back to tracks[0]
// but drop such tracks for now.
let (decoder, track_id) =
decoder.ok_or(SymphError::DecodeError("no compatible track found"))?;
let p = Parsed {
format,
decoder,
track_id,
meta,
supports_backseek,
};
out = LiveInput::Parsed(p);
}
Ok(out)
}
/// Returns a reference to the data parsed from this input stream, if it has
/// been made available via [`Self::promote`].
#[must_use]
pub fn parsed(&self) -> Option<&Parsed> {
if let Self::Parsed(parsed) = self {
Some(parsed)
} else {
None
}
}
/// Returns a mutable reference to the data parsed from this input stream, if it
/// has been made available via [`Self::promote`].
pub fn parsed_mut(&mut self) -> Option<&mut Parsed> {
if let Self::Parsed(parsed) = self {
Some(parsed)
} else {
None
}
}
/// Returns whether this stream's headers have been fully parsed, and so whether
/// the track can be played or have its metadata read.
#[must_use]
pub fn is_playable(&self) -> bool {
self.parsed().is_some()
}
/// Tries to get any information about this audio stream acquired during parsing.
///
/// Only exists when this input is [`LiveInput::Parsed`].
pub fn metadata(&mut self) -> Result<Metadata<'_>, MetadataError> {
if let Some(parsed) = self.parsed_mut() {
Ok(parsed.into())
} else {
Err(MetadataError::NotParsed)
}
}
}
#[cfg(test)]
mod tests {
use crate::{
constants::test_data::FILE_VID_TARGET,
input::{codecs::*, File, Input},
};
#[tokio::test]
#[ntest::timeout(10_000)]
async fn promote_finds_valid_audio() {
// Video files often set their default to... the video stream, unsurprisingly.
// In these cases we still want to play the attached audio -- this checks that songbird
// finds the audio on a non-default track via `LiveInput::promote`.
let input = Input::from(File::new(FILE_VID_TARGET));
input
.make_playable_async(get_codec_registry(), get_probe())
.await
.unwrap();
}
}

View File

@@ -0,0 +1,175 @@
use super::AuxMetadata;
use serde::{Deserialize, Serialize};
use serde_aux::prelude::*;
use std::{collections::HashMap, time::Duration};
// These have been put together by looking at ffprobe's output
// and the canonical data formats given in
// https://github.com/FFmpeg/FFmpeg/blob/master/doc/ffprobe.xsd
#[derive(Deserialize, Serialize)]
pub struct Output {
pub streams: Vec<Stream>,
pub format: Format,
}
#[derive(Deserialize, Serialize)]
pub struct Stream {
pub index: u64,
pub codec_name: Option<String>,
pub codec_long_name: Option<String>,
pub profile: Option<String>,
pub codec_type: Option<String>,
pub codec_tag: String,
pub codec_tag_string: String,
pub extradata: Option<String>,
pub extradata_size: Option<u64>,
pub extradata_hash: Option<String>,
// Video attributes skipped.
pub sample_fmt: Option<String>,
#[serde(deserialize_with = "deserialize_option_number_from_string")]
pub sample_rate: Option<u32>,
pub channels: Option<u32>,
pub channel_layout: Option<String>,
pub bits_per_sample: Option<u32>,
pub id: Option<String>,
pub r_frame_rate: String,
pub avg_frame_rate: String,
pub time_base: String,
pub start_pts: Option<i64>,
#[serde(deserialize_with = "deserialize_option_number_from_string")]
pub start_time: Option<f64>,
pub duration_ts: Option<u64>,
#[serde(deserialize_with = "deserialize_option_number_from_string")]
pub duration: Option<f64>,
#[serde(deserialize_with = "deserialize_option_number_from_string")]
pub bit_rate: Option<u64>,
#[serde(deserialize_with = "deserialize_option_number_from_string")]
pub max_bit_rate: Option<u64>,
pub bits_per_raw_sample: Option<u64>,
pub nb_frames: Option<u64>,
pub nb_read_frames: Option<u64>,
pub nb_read_packets: Option<u64>,
// Side Data List skipped.
pub disposition: Option<Disposition>,
pub tags: Option<HashMap<String, String>>,
}
#[allow(clippy::struct_excessive_bools)]
#[derive(Deserialize, Serialize)]
pub struct Disposition {
#[serde(deserialize_with = "deserialize_bool_from_anything")]
pub default: bool,
#[serde(deserialize_with = "deserialize_bool_from_anything")]
pub dub: bool,
#[serde(deserialize_with = "deserialize_bool_from_anything")]
pub original: bool,
#[serde(deserialize_with = "deserialize_bool_from_anything")]
pub comment: bool,
#[serde(deserialize_with = "deserialize_bool_from_anything")]
pub lyrics: bool,
#[serde(deserialize_with = "deserialize_bool_from_anything")]
pub karaoke: bool,
#[serde(deserialize_with = "deserialize_bool_from_anything")]
pub forced: bool,
#[serde(deserialize_with = "deserialize_bool_from_anything")]
pub hearing_impaired: bool,
#[serde(deserialize_with = "deserialize_bool_from_anything")]
pub visual_impaired: bool,
#[serde(deserialize_with = "deserialize_bool_from_anything")]
pub clean_effects: bool,
#[serde(deserialize_with = "deserialize_bool_from_anything")]
pub attached_pic: bool,
#[serde(deserialize_with = "deserialize_bool_from_anything")]
pub timed_thumbnails: bool,
#[serde(deserialize_with = "deserialize_bool_from_anything")]
pub captions: bool,
#[serde(deserialize_with = "deserialize_bool_from_anything")]
pub descriptions: bool,
#[serde(deserialize_with = "deserialize_bool_from_anything")]
pub metadata: bool,
#[serde(deserialize_with = "deserialize_bool_from_anything")]
pub dependent: bool,
#[serde(deserialize_with = "deserialize_bool_from_anything")]
pub still_image: bool,
}
#[derive(Deserialize, Serialize)]
pub struct Format {
pub filename: String,
pub nb_streams: u64,
pub nb_programs: u64,
#[serde(rename = "format_name")]
pub name: String,
#[serde(rename = "format_long_name")]
pub long_name: Option<String>,
#[serde(deserialize_with = "deserialize_option_number_from_string")]
pub start_time: Option<f64>,
#[serde(deserialize_with = "deserialize_option_number_from_string")]
pub duration: Option<f64>,
#[serde(deserialize_with = "deserialize_option_number_from_string")]
pub size: Option<u64>,
#[serde(deserialize_with = "deserialize_option_number_from_string")]
pub bit_rate: Option<u64>,
pub probe_score: i64,
pub tags: Option<HashMap<String, String>>,
}
fn apply_tags(tag_map: HashMap<String, String>, dest: &mut AuxMetadata) {
for (k, v) in tag_map {
match k.as_str().to_lowercase().as_str() {
"title" => dest.title = Some(v),
"album" => dest.album = Some(v),
"artist" => dest.artist = Some(v),
"date" => dest.date = Some(v),
"channels" =>
if let Ok(chans) = str::parse::<u8>(&v) {
dest.channels = Some(chans);
},
"sample_rate" =>
if let Ok(samples) = str::parse::<u32>(&v) {
dest.sample_rate = Some(samples);
},
_ => {},
}
}
}
impl Output {
pub fn into_aux_metadata(self) -> AuxMetadata {
let duration = self.format.duration.map(Duration::from_secs_f64);
let start_time = self
.format
.duration
.map(|v| v.max(0.0))
.map(Duration::from_secs_f64);
let mut out = AuxMetadata {
start_time,
duration,
..AuxMetadata::default()
};
if let Some(tags) = self.format.tags {
apply_tags(tags, &mut out);
}
for stream in self.streams {
if stream.codec_type.as_deref() != Some("audio") {
continue;
}
if let Some(tags) = stream.tags {
apply_tags(tags, &mut out);
}
}
out
}
}

View File

@@ -0,0 +1,113 @@
//! Metadata formats specific to [`crate::input::Compose`] types.
use crate::error::JsonError;
use std::time::Duration;
use symphonia_core::{meta::Metadata as ContainerMetadata, probe::ProbedMetadata};
pub(crate) mod ffprobe;
mod ytdl;
pub use ytdl::Output as YoutubeDlOutput;
use super::Parsed;
/// Extra information about an [`Input`] which is acquired without
/// parsing the file itself (e.g., from a webpage).
///
/// You can access this via [`Input::aux_metadata`] and [`Compose::aux_metadata`].
///
/// [`Input`]: crate::input::Input
/// [`Input::aux_metadata`]: crate::input::Input::aux_metadata
/// [`Compose::aux_metadata`]: crate::input::Compose::aux_metadata
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct AuxMetadata {
/// The track name of this stream.
pub track: Option<String>,
/// The main artist of this stream.
pub artist: Option<String>,
/// The album name of this stream.
pub album: Option<String>,
/// The date of creation of this stream.
pub date: Option<String>,
/// The number of audio channels in this stream.
pub channels: Option<u8>,
/// The YouTube channel of this stream.
pub channel: Option<String>,
/// The time at which the first true sample is played back.
///
/// This occurs as an artefact of coder delay.
pub start_time: Option<Duration>,
/// The reported duration of this stream.
pub duration: Option<Duration>,
/// The sample rate of this stream.
pub sample_rate: Option<u32>,
/// The source url of this stream.
pub source_url: Option<String>,
/// The YouTube title of this stream.
pub title: Option<String>,
/// The thumbnail url of this stream.
pub thumbnail: Option<String>,
}
impl AuxMetadata {
/// Extract metadata and details from the output of `ffprobe -of json`.
pub fn from_ffprobe_json(value: &mut [u8]) -> Result<Self, JsonError> {
let output: ffprobe::Output = serde_json::from_slice(value)?;
Ok(output.into_aux_metadata())
}
/// Move all fields from an [`AuxMetadata`] object into a new one.
#[must_use]
pub fn take(&mut self) -> Self {
Self {
track: self.track.take(),
artist: self.artist.take(),
album: self.album.take(),
date: self.date.take(),
channels: self.channels.take(),
channel: self.channel.take(),
start_time: self.start_time.take(),
duration: self.duration.take(),
sample_rate: self.sample_rate.take(),
source_url: self.source_url.take(),
title: self.title.take(),
thumbnail: self.thumbnail.take(),
}
}
}
/// In-stream information about an [`Input`] acquired by parsing an audio file.
///
/// To access this, the [`Input`] must be made live and parsed by symphonia. To do
/// this, you can:
/// * Pre-process the track in your own code using [`Input::make_playable`], and
/// then [`Input::metadata`].
/// * Use [`TrackHandle::action`] to access the track's metadata via [`View`],
/// *if the track has started or been made playable*.
///
/// You probably want to use [`AuxMetadata`] instead; this requires a live track,
/// which has higher memory use for buffers etc.
///
/// [`Input`]: crate::input::Input
/// [`Input::make_playable`]: super::Input::make_playable
/// [`Input::metadata`]: super::Input::metadata
/// [`TrackHandle::action`]: crate::tracks::TrackHandle::action
/// [`View`]: crate::tracks::View
pub struct Metadata<'a> {
/// Metadata found while probing for the format of an [`Input`] (e.g., ID3 tags).
///
/// [`Input`]: crate::input::Input
pub probe: &'a mut ProbedMetadata,
/// Metadata found inside the format/container of an audio stream.
pub format: ContainerMetadata<'a>,
}
impl<'a> From<&'a mut Parsed> for Metadata<'a> {
fn from(val: &'a mut Parsed) -> Self {
Metadata {
probe: &mut val.meta,
format: val.format.metadata(),
}
}
}

View File

@@ -0,0 +1,77 @@
//! `YoutubeDl` track metadata.
use super::AuxMetadata;
use crate::constants::SAMPLE_RATE_RAW;
use serde::{Deserialize, Serialize};
use std::{collections::HashMap, time::Duration};
/// Information returned by yt-dlp about a URL.
///
/// Returned by [`crate::input::YoutubeDl::query`].
#[derive(Deserialize, Serialize, Debug)]
pub struct Output {
/// The main artist.
pub artist: Option<String>,
/// The album name.
pub album: Option<String>,
/// The channel name.
pub channel: Option<String>,
/// The duration of the stream in seconds.
pub duration: Option<f64>,
/// The size of the stream.
pub filesize: Option<u64>,
/// Required HTTP headers to fetch the track stream.
pub http_headers: Option<HashMap<String, String>>,
/// Release date of this track.
pub release_date: Option<String>,
/// The thumbnail URL for this track.
pub thumbnail: Option<String>,
/// The title of this track.
pub title: Option<String>,
/// The track name.
pub track: Option<String>,
/// The date this track was uploaded on.
pub upload_date: Option<String>,
/// The name of the uploader.
pub uploader: Option<String>,
/// The stream URL.
pub url: String,
/// The URL of the public-facing webpage for this track.
pub webpage_url: Option<String>,
/// The stream protocol.
pub protocol: Option<String>,
}
impl Output {
/// Requests auxiliary metadata which can be accessed without parsing the file.
pub fn as_aux_metadata(&self) -> AuxMetadata {
let album = self.album.clone();
let track = self.track.clone();
let true_artist = self.artist.as_ref();
let artist = true_artist.or(self.uploader.as_ref()).cloned();
let r_date = self.release_date.as_ref();
let date = r_date.or(self.upload_date.as_ref()).cloned();
let channel = self.channel.clone();
let duration = self.duration.map(Duration::from_secs_f64);
let source_url = self.webpage_url.clone();
let title = self.title.clone();
let thumbnail = self.thumbnail.clone();
AuxMetadata {
track,
artist,
album,
date,
channels: Some(2),
channel,
duration,
sample_rate: Some(SAMPLE_RATE_RAW as u32),
source_url,
title,
thumbnail,
..AuxMetadata::default()
}
}
}

381
app/vendor/songbird/src/input/mod.rs vendored Normal file
View File

@@ -0,0 +1,381 @@
//! Raw audio input data streams and sources.
//!
//! [`Input`]s in Songbird are based on [symphonia], which provides demuxing,
//! decoding and management of synchronous byte sources (i.e., any items which
//! `impl` [`Read`]).
//!
//! Songbird adds support for the Opus codec to symphonia via [`OpusDecoder`],
//! the [DCA1] file format via [`DcaReader`], and a simple PCM adapter via [`RawReader`];
//! the [format] and [codec registries] in [`codecs`] install these on top of those
//! enabled in your `Cargo.toml` when you include symphonia.
//!
//! ## Common sources
//! * Any owned byte slice: `&'static [u8]`, `Bytes`, or `Vec<u8>`,
//! * [`File`] offers a lazy way to open local audio files,
//! * [`HttpRequest`] streams a given file from a URL using the reqwest HTTP library,
//! * [`YoutubeDl`] uses `yt-dlp` (or any other `youtube-dl`-like program) to scrape
//! a target URL for a usable audio stream, before opening an [`HttpRequest`].
//!
//! ## Adapters
//! Songbird includes several adapters to make developing your own inputs easier:
//! * [`cached::*`], which allow seeking and shared caching of an input stream (storing
//! it in memory in a variety of formats),
//! * [`ChildContainer`] for managing audio given by a process chain,
//! * [`RawAdapter`], for feeding in a synchronous `f32`-PCM stream, and
//! * [`AsyncAdapterStream`], for passing bytes from an `AsyncRead` (`+ AsyncSeek`) stream
//! into the mixer.
//!
//! ## Opus frame passthrough.
//! Some sources, such as [`Compressed`] or any WebM/Opus/DCA file, support
//! direct frame passthrough to the driver. This lets you directly send the
//! audio data you have *without decoding, re-encoding, or mixing*. In many
//! cases, this can greatly reduce the CPU cost required by the driver.
//!
//! This functionality requires that:
//! * only one track is active (including paused tracks),
//! * that track's input supports direct Opus frame reads,
//! * this input's frames are all sized to 20ms.
//! * and that track's volume is set to `1.0`.
//!
//! [`Input`]s which are almost suitable but which have **any** illegal frames will be
//! blocked from passthrough to prevent glitches such as repeated encoder frame gaps.
//!
//! [symphonia]: https://docs.rs/symphonia
//! [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html
//! [`Compressed`]: cached::Compressed
//! [DCA1]: https://github.com/bwmarrin/dca
//! [`cached::*`]: cached
//! [`OpusDecoder`]: codecs::OpusDecoder
//! [`DcaReader`]: codecs::DcaReader
//! [`RawReader`]: codecs::RawReader
//! [format]: codecs::get_probe
//! [codec registries]: codecs::get_codec_registry
mod adapters;
mod audiostream;
pub mod codecs;
mod compose;
mod error;
#[cfg(test)]
pub mod input_tests;
mod live_input;
pub mod metadata;
mod parsed;
mod sources;
pub mod utils;
pub use self::{
adapters::*,
audiostream::*,
compose::*,
error::*,
live_input::*,
metadata::{AuxMetadata, Metadata},
parsed::*,
sources::*,
};
pub use symphonia_core as core;
use std::{error::Error, io::Cursor};
use symphonia_core::{codecs::CodecRegistry, probe::Probe};
use tokio::runtime::Handle as TokioHandle;
/// An audio source, which can be live or lazily initialised.
///
/// This can be created from a wide variety of sources:
/// * Any owned byte slice: `&'static [u8]`, `Bytes`, or `Vec<u8>`,
/// * [`File`] offers a lazy way to open local audio files,
/// * [`HttpRequest`] streams a given file from a URL using the reqwest HTTP library,
/// * [`YoutubeDl`] uses `yt-dlp` (or any other `youtube-dl`-like program) to scrape
/// a target URL for a usable audio stream, before opening an [`HttpRequest`].
///
/// Any [`Input`] (or struct with `impl Into<Input>`) can also be made into a [`Track`] via
/// `From`/`Into`.
///
/// # Example
///
/// ```
/// # use tokio::runtime;
/// #
/// # let basic_rt = runtime::Builder::new_current_thread().enable_io().build().unwrap();
/// # basic_rt.block_on(async {
/// use songbird::{
/// driver::Driver,
/// input::{codecs::*, Compose, Input, MetadataError, YoutubeDl},
/// tracks::Track,
/// };
/// // Inputs are played using a `Driver`, or `Call`.
/// let mut driver = Driver::new(Default::default());
///
/// // Lazy inputs take very little resources, and don't occupy any resources until we
/// // need to play them (by default).
/// let mut lazy = YoutubeDl::new(
/// reqwest::Client::new(),
/// // Referenced under CC BY-NC-SA 3.0 -- https://creativecommons.org/licenses/by-nc-sa/3.0/
/// "https://cloudkicker.bandcamp.com/track/94-days",
/// );
/// let lazy_c = lazy.clone();
///
/// // With sources like `YoutubeDl`, we can get metadata from, e.g., a track's page.
/// let aux_metadata = lazy.aux_metadata().await.unwrap();
/// assert_eq!(aux_metadata.track, Some("94 Days".to_string()));
///
/// // Once we pass an `Input` to the `Driver`, we can only remotely control it via
/// // a `TrackHandle`.
/// let handle = driver.play_input(lazy.into());
///
/// // We can also modify some of its initial state via `Track`s.
/// let handle = driver.play(Track::from(lazy_c).volume(0.5).pause());
///
/// // In-memory sources like `Vec<u8>`, or `&'static [u8]` are easy to use, and only take a
/// // little time for the mixer to parse their headers.
/// // You can also use the adapters in `songbird::input::cached::*`to keep a source
/// // from the Internet, HTTP, or a File in-memory *and* share it among calls.
/// let in_memory = include_bytes!("../../resources/ting.mp3");
/// let mut in_memory_input = in_memory.into();
///
/// // This source is live...
/// assert!(matches!(in_memory_input, Input::Live(..)));
/// // ...but not yet playable, and we can't access its `Metadata`.
/// assert!(!in_memory_input.is_playable());
/// assert!(matches!(in_memory_input.metadata(), Err(MetadataError::NotParsed)));
///
/// // If we want to inspect metadata (and we can't use AuxMetadata for any reason), we have
/// // to parse the track ourselves.
/// //
/// // We can access it on a live track using `TrackHandle::action()`.
/// in_memory_input = in_memory_input
/// .make_playable_async(get_codec_registry(), get_probe())
/// .await
/// .expect("WAV support is included, and this file is good!");
///
/// // Symphonia's metadata can be difficult to use: prefer `AuxMetadata` when you can!
/// use symphonia_core::meta::{StandardTagKey, Value};
/// let mut metadata = in_memory_input.metadata();
/// let meta = metadata.as_mut().unwrap();
/// let mut probed = meta.probe.get().unwrap();
///
/// let track_name = probed
/// .current().unwrap()
/// .tags().iter().filter(|v| v.std_key == Some(StandardTagKey::TrackTitle))
/// .next().unwrap();
/// if let Value::String(s) = &track_name.value {
/// assert_eq!(s, "Ting!");
/// } else { panic!() };
///
/// // ...and these are played like any other input.
/// let handle = driver.play_input(in_memory_input);
/// # });
/// ```
///
/// [`Track`]: crate::tracks::Track
pub enum Input {
/// A byte source which is not yet initialised.
///
/// When a parent track is either played or explicitly readied, the inner [`Compose`]
/// is used to create an [`Input::Live`].
Lazy(
/// A trait object which can be used to (re)create a usable byte stream.
Box<dyn Compose>,
),
/// An initialised byte source.
///
/// This contains a raw byte stream, the lazy initialiser that was used,
/// as well as any symphonia-specific format data and/or hints.
Live(
/// The byte source, plus symphonia-specific data.
LiveInput,
/// The struct used to initialise this source, if available.
///
/// This is used to recreate the stream when a source does not support
/// backward seeking, if present.
Option<Box<dyn Compose>>,
),
}
impl Input {
/// Requests auxiliary metadata which can be accessed without parsing the file.
///
/// This method will never be called by songbird but allows, for instance, access to metadata
/// which might only be visible to a web crawler, e.g., uploader or source URL.
///
/// This requires that the [`Input`] has a [`Compose`] available to use, otherwise it
/// will always fail with [`AudioStreamError::Unsupported`].
pub async fn aux_metadata(&mut self) -> Result<AuxMetadata, AuxMetadataError> {
match self {
Self::Lazy(ref mut composer) | Self::Live(_, Some(ref mut composer)) =>
composer.aux_metadata().await.map_err(Into::into),
Self::Live(_, None) => Err(AuxMetadataError::NoCompose),
}
}
/// Tries to get any information about this audio stream acquired during parsing.
///
/// Only exists when this input is both [`Self::Live`] and has been fully parsed.
/// In general, you probably want to use [`Self::aux_metadata`].
pub fn metadata(&mut self) -> Result<Metadata<'_>, MetadataError> {
if let Self::Live(live, _) = self {
live.metadata()
} else {
Err(MetadataError::NotLive)
}
}
/// Initialises (but does not parse) an [`Input::Lazy`] into an [`Input::Live`],
/// placing blocking I/O on the current thread.
///
/// This requires a [`TokioHandle`] to a tokio runtime to spawn any `async` sources.
///
/// *This is a blocking operation. If you wish to use this from an async task, you
/// must do so via [`Self::make_live_async`].*
///
/// This is a no-op for an [`Input::Live`].
pub fn make_live(self, handle: &TokioHandle) -> Result<Self, AudioStreamError> {
if let Self::Lazy(mut lazy) = self {
let (created, lazy) = if lazy.should_create_async() {
let (tx, rx) = flume::bounded(1);
handle.spawn(async move {
let out = lazy.create_async().await;
drop(tx.send_async((out, lazy)));
});
rx.recv().map_err(|_| {
let err_msg: Box<dyn Error + Send + Sync> =
"async Input create handler panicked".into();
AudioStreamError::Fail(err_msg)
})?
} else {
(lazy.create(), lazy)
};
Ok(Self::Live(LiveInput::Raw(created?), Some(lazy)))
} else {
Ok(self)
}
}
/// Initialises (but does not parse) an [`Input::Lazy`] into an [`Input::Live`],
/// placing blocking I/O on the a `spawn_blocking` executor.
///
/// This is a no-op for an [`Input::Live`].
pub async fn make_live_async(self) -> Result<Self, AudioStreamError> {
if let Self::Lazy(mut lazy) = self {
let (created, lazy) = if lazy.should_create_async() {
(lazy.create_async().await, lazy)
} else {
tokio::task::spawn_blocking(move || (lazy.create(), lazy))
.await
.map_err(|_| {
let err_msg: Box<dyn Error + Send + Sync> =
"synchronous Input create handler panicked".into();
AudioStreamError::Fail(err_msg)
})?
};
Ok(Self::Live(LiveInput::Raw(created?), Some(lazy)))
} else {
Ok(self)
}
}
/// Initialises and parses an [`Input::Lazy`] into an [`Input::Live`],
/// placing blocking I/O on the current thread.
///
/// This requires a [`TokioHandle`] to a tokio runtime to spawn any `async` sources.
/// If you can't access one, then consider manually using [`LiveInput::promote`].
///
/// *This is a blocking operation. Symphonia uses standard library I/O (e.g., [`Read`], [`Seek`]).
/// If you wish to use this from an async task, you must do so within `spawn_blocking`.*
///
/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html
/// [`Seek`]: https://doc.rust-lang.org/std/io/trait.Seek.html
pub fn make_playable(
self,
codecs: &CodecRegistry,
probe: &Probe,
handle: &TokioHandle,
) -> Result<Self, MakePlayableError> {
let out = self.make_live(handle)?;
match out {
Self::Lazy(_) => unreachable!(),
Self::Live(input, lazy) => {
let promoted = input.promote(codecs, probe)?;
Ok(Self::Live(promoted, lazy))
},
}
}
/// Initialises and parses an [`Input::Lazy`] into an [`Input::Live`],
/// placing blocking I/O on a tokio blocking thread.
pub async fn make_playable_async(
self,
codecs: &'static CodecRegistry,
probe: &'static Probe,
) -> Result<Self, MakePlayableError> {
let out = self.make_live_async().await?;
match out {
Self::Lazy(_) => unreachable!(),
Self::Live(input, lazy) => {
let promoted = tokio::task::spawn_blocking(move || input.promote(codecs, probe))
.await
.map_err(|_| MakePlayableError::Panicked)??;
Ok(Self::Live(promoted, lazy))
},
}
}
/// Returns whether this audio stream is full initialised, parsed, and
/// ready to play (e.g., `Self::Live(LiveInput::Parsed(p), _)`).
#[must_use]
pub fn is_playable(&self) -> bool {
if let Self::Live(input, _) = self {
input.is_playable()
} else {
false
}
}
/// Returns a reference to the live input, if it has been created via
/// [`Self::make_live`] or [`Self::make_live_async`].
#[must_use]
pub fn live(&self) -> Option<&LiveInput> {
if let Self::Live(input, _) = self {
Some(input)
} else {
None
}
}
/// Returns a mutable reference to the live input, if it been created via
/// [`Self::make_live`] or [`Self::make_live_async`].
pub fn live_mut(&mut self) -> Option<&mut LiveInput> {
if let Self::Live(ref mut input, _) = self {
Some(input)
} else {
None
}
}
/// Returns a reference to the data parsed from this input stream, if it has
/// been made available via [`Self::make_playable`] or [`LiveInput::promote`].
#[must_use]
pub fn parsed(&self) -> Option<&Parsed> {
self.live().and_then(LiveInput::parsed)
}
/// Returns a mutable reference to the data parsed from this input stream, if it
/// has been made available via [`Self::make_playable`] or [`LiveInput::promote`].
pub fn parsed_mut(&mut self) -> Option<&mut Parsed> {
self.live_mut().and_then(LiveInput::parsed_mut)
}
}
impl<T: AsRef<[u8]> + Send + Sync + 'static> From<T> for Input {
fn from(val: T) -> Self {
let raw_src = LiveInput::Raw(AudioStream {
input: Box::new(Cursor::new(val)),
});
Input::Live(raw_src, None)
}
}

31
app/vendor/songbird/src/input/parsed.rs vendored Normal file
View File

@@ -0,0 +1,31 @@
use symphonia_core::{codecs::Decoder, formats::FormatReader, probe::ProbedMetadata};
/// An audio file which has had its headers parsed and decoder state built.
pub struct Parsed {
/// Audio packet, seeking, and state access for all tracks in a file.
///
/// This may be used to access packets one at a time from the input file.
/// Additionally, this exposes container-level and per track metadata which
/// have been extracted.
pub format: Box<dyn FormatReader>,
/// Decoder state for the chosen track.
pub decoder: Box<dyn Decoder>,
/// The chosen track's ID.
///
/// This is required to identify the correct packet stream inside the container.
pub track_id: u32,
/// Metadata extracted by symphonia while detecting a file's format.
///
/// Typically, this detects metadata *outside* the file's core format (i.e.,
/// ID3 tags in MP3 and WAV files).
pub meta: ProbedMetadata,
/// Whether the contained format supports arbitrary seeking.
///
/// If set to false, Songbird will attempt to recreate the input if
/// it must seek backwards.
pub supports_backseek: bool,
}

View File

@@ -0,0 +1,77 @@
use crate::input::{AudioStream, AudioStreamError, Compose, Input};
use std::{error::Error, path::Path};
use symphonia_core::io::MediaSource;
/// A lazily instantiated local file.
#[derive(Clone, Debug)]
pub struct File<P: AsRef<Path>> {
path: P,
}
impl<P: AsRef<Path>> File<P> {
/// Creates a lazy file object, which will open the target path.
///
/// This is infallible as the path is only checked during creation.
pub fn new(path: P) -> Self {
Self { path }
}
}
impl<P: AsRef<Path> + Send + Sync + 'static> From<File<P>> for Input {
fn from(val: File<P>) -> Self {
Input::Lazy(Box::new(val))
}
}
#[async_trait::async_trait]
impl<P: AsRef<Path> + Send + Sync> Compose for File<P> {
fn create(&mut self) -> Result<AudioStream<Box<dyn MediaSource>>, AudioStreamError> {
let err: Box<dyn Error + Send + Sync> =
"Files should be created asynchronously.".to_string().into();
Err(AudioStreamError::Fail(err))
}
async fn create_async(
&mut self,
) -> Result<AudioStream<Box<dyn MediaSource>>, AudioStreamError> {
let file = tokio::fs::File::open(&self.path)
.await
.map_err(|io| AudioStreamError::Fail(Box::new(io)))?;
let input = Box::new(file.into_std().await);
Ok(AudioStream { input })
}
fn should_create_async(&self) -> bool {
true
}
// SEE: issue #186
// Below is removed due to issues with:
// 1) deadlocks on small files.
// 2) serde_aux poorly handles missing field names.
//
// Probes for metadata about this audio file using `ffprobe`.
// async fn aux_metadata(&mut self) -> Result<AuxMetadata, AudioStreamError> {
// let args = [
// "-v",
// "quiet",
// "-of",
// "json",
// "-show_format",
// "-show_streams",
// "-i",
// ];
// let mut output = Command::new("ffprobe")
// .args(args)
// .arg(self.path.as_ref().as_os_str())
// .output()
// .await
// .map_err(|e| AudioStreamError::Fail(Box::new(e)))?;
// AuxMetadata::from_ffprobe_json(&mut output.stdout[..])
// .map_err(|e| AudioStreamError::Fail(Box::new(e)))
// }
}

Some files were not shown because too many files have changed in this diff Show More