Compare commits
53 Commits
app-v0.1.3
...
app-v0.1.5
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b64097ef0d | ||
|
|
81bd8bd573 | ||
|
|
e716065538 | ||
|
|
41f539dd48 | ||
|
|
b30aa4a0db | ||
|
|
835b547ca2 | ||
|
|
09bdf05762 | ||
|
|
251f5e116d | ||
|
|
c1b561d66d | ||
|
|
9a527aa470 | ||
|
|
628d89c4f1 | ||
|
|
478968949c | ||
|
|
caad09c613 | ||
|
|
97b827f424 | ||
|
|
045218b008 | ||
|
|
e32671f871 | ||
|
|
e23a962923 | ||
|
|
716b5bb2e6 | ||
|
|
e88c94d934 | ||
|
|
4906d8e7dd | ||
|
|
844958d311 | ||
|
|
3af881935c | ||
|
|
20bd90f65e | ||
|
|
9a296a5683 | ||
|
|
5767ffa5ae | ||
|
|
378ccfafad | ||
|
|
999c432216 | ||
|
|
218e7734be | ||
|
|
912b3304ac | ||
|
|
decd7ae477 | ||
|
|
2a57b2b2f4 | ||
|
|
ba44cc224c | ||
|
|
2e00104332 | ||
|
|
ade76d2585 | ||
|
|
39ac20682e | ||
|
|
7539be07a5 | ||
|
|
21eeb2e8bf | ||
|
|
b58fc149d4 | ||
|
|
a59ed43ae1 | ||
|
|
3e4974f3bd | ||
|
|
4d83543acf | ||
|
|
9c1e82a890 | ||
|
|
f05424e1b2 | ||
|
|
be09cfad81 | ||
|
|
ac633c63ca | ||
|
|
499280504e | ||
|
|
c816358552 | ||
|
|
9f5f3d891f | ||
|
|
04c23f9bf7 | ||
|
|
49050b18a9 | ||
|
|
9f62bfb059 | ||
|
|
e428aad58a | ||
|
|
a665dd8d21 |
@@ -18,7 +18,7 @@
|
||||
- **빠른 음성 대화**: 말하는 도중에 전사하고 답은 문장 단위로 바로 읽어서, 말이 끝난 뒤 첫 소리까지 약 1~1.5초를 목표로 합니다.
|
||||
- **최대 10명 동시 대화**: 화자별로 따로 받아 전사합니다. 여러 명이 동시에 부를 때 어떻게 답할지(처음 대화만 / 들어온 순서대로 / 마지막 대화만 / 제일 중요한 대화만)는 GUI에서 고릅니다.
|
||||
- **호출어와 지정한 사람**:
|
||||
- 호출어는 GUI에서 여러 개 등록합니다.
|
||||
- 호출어는 GUI에서 여러 개 등록합니다. 기본값은 "비서", "비서야", "비서님", "김비서", "김비서야", "김비서님"입니다.
|
||||
- "지정한 사람만 응답"과 "지정한 사람은 호출어 없이 대답"을 각각 켜고 끕니다.
|
||||
- **AI 채팅 채널**: 등록한 디스코드 채팅 채널을 자비스가 읽고, 메시지를 보내고, 자기 메시지를 고칩니다. 채널 ID, 이름, 설명은 GUI에서 편집하고, 고친 이름·설명은 실제 디스코드 채널 이름·주제에도 반영합니다(끌 수 있음).
|
||||
- **예약 알림과 정기 작업**: "10분 뒤에 알려 줘", "내일 7시에 회의 알려 줘", "평일 아침 9시에 날씨 알려 줘"처럼 말이나 채팅으로 맡깁니다. 맡긴 사람이 자비스와 같은 음성 채널에 있으면 음성으로, 아니면 채팅 채널에 글로 전합니다. 정기 작업은 날씨, 웹 검색처럼 찾아보는 기능만 쓰고, 관리 앱 "예약" 화면에서 목록을 보고 지울 수 있습니다.
|
||||
|
||||
4
app/Cargo.lock
generated
4
app/Cargo.lock
generated
@@ -1520,7 +1520,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.32"
|
||||
version = "0.1.51"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -3324,6 +3324,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "songbird"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "371362adce10df3244fbf93475ac53391aedd425482f160a9dbb3e042577de8b"
|
||||
dependencies = [
|
||||
"aead",
|
||||
"aes-gcm",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.32"
|
||||
version = "0.1.51"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"
|
||||
@@ -44,17 +44,13 @@ jpeg-decoder = "0.3"
|
||||
name = "vdisplay_elevated"
|
||||
harness = false
|
||||
|
||||
# songbird 0.6.1 with the Jarvis patch for the first packets of a speaker (vendor/songbird/JARVIS-PATCH.md).
|
||||
[patch.crates-io]
|
||||
songbird = { path = "vendor/songbird" }
|
||||
|
||||
[profile.release]
|
||||
lto = "thin"
|
||||
codegen-units = 4
|
||||
strip = "symbols"
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Globalization", "Win32_Security_Cryptography", "Win32_System_SystemInformation", "Win32_Devices_DeviceAndDriverInstallation", "Win32_Graphics_Gdi", "Win32_UI_WindowsAndMessaging", "Win32_System_Registry", "Win32_UI_Shell", "Win32_System_Threading", "Win32_System_Performance", "Win32_System_Diagnostics_ToolHelp", "Win32_Graphics_Dwm"] }
|
||||
windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Globalization", "Win32_Security", "Win32_Security_Cryptography", "Win32_System_SystemInformation", "Win32_Devices_DeviceAndDriverInstallation", "Win32_Graphics_Gdi", "Win32_UI_WindowsAndMessaging", "Win32_UI_Input_KeyboardAndMouse", "Win32_System_Registry", "Win32_UI_Shell", "Win32_System_Threading", "Win32_System_Performance", "Win32_System_Diagnostics_ToolHelp", "Win32_Graphics_Dwm"] }
|
||||
windows = { version = "0.62", features = ["Foundation", "Graphics", "Graphics_Capture", "Graphics_DirectX", "Graphics_DirectX_Direct3D11", "Win32_Foundation", "Win32_Graphics_Gdi", "Win32_Graphics_Direct3D", "Win32_Graphics_Direct3D11", "Win32_Graphics_Dxgi", "Win32_Graphics_Dxgi_Common", "Win32_System_WinRT", "Win32_System_WinRT_Direct3D11", "Win32_System_WinRT_Graphics_Capture"] }
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
|
||||
6
app/shell/Cargo.lock
generated
6
app/shell/Cargo.lock
generated
@@ -2637,7 +2637,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.32"
|
||||
version = "0.1.51"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -2666,7 +2666,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.32"
|
||||
version = "0.1.51"
|
||||
dependencies = [
|
||||
"jarvis-app",
|
||||
"serde_json",
|
||||
@@ -5562,6 +5562,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "songbird"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "371362adce10df3244fbf93475ac53391aedd425482f160a9dbb3e042577de8b"
|
||||
dependencies = [
|
||||
"aead",
|
||||
"aes-gcm",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.32"
|
||||
version = "0.1.51"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"
|
||||
@@ -21,10 +21,6 @@ tauri-plugin-single-instance = "2"
|
||||
serde_json = "1"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
|
||||
# songbird 0.6.1 with the Jarvis patch for the first packets of a speaker (vendor/songbird/JARVIS-PATCH.md).
|
||||
[patch.crates-io]
|
||||
songbird = { path = "../vendor/songbird" }
|
||||
|
||||
[profile.release]
|
||||
lto = "thin"
|
||||
codegen-units = 4
|
||||
|
||||
@@ -226,6 +226,8 @@ Section "Jarvis (필수)" SecMain
|
||||
SetOutPath "$INSTDIR"
|
||||
File "${SRC}/jarvis.exe"
|
||||
File "${SRC}/WebView2Loader.dll"
|
||||
; PC control's agent-desktop (the app checks its hash before every start).
|
||||
File "${SRC}/agent-desktop.exe"
|
||||
; The shortcuts' icon, named by its content: Windows keeps an icon it has shown by the file's path,
|
||||
; so a changed icon under the old path would stay the old one on the desktop and in search.
|
||||
Delete "$INSTDIR\jarvis-*.ico"
|
||||
@@ -334,6 +336,7 @@ Section "Uninstall"
|
||||
; folder the person chose may hold other things; folders go only when empty.
|
||||
Delete "$INSTDIR\jarvis.exe"
|
||||
Delete "$INSTDIR\WebView2Loader.dll"
|
||||
Delete "$INSTDIR\agent-desktop.exe"
|
||||
Delete "$INSTDIR\jarvis-*.ico"
|
||||
!include "${UNINSTALL_LIST}"
|
||||
Delete "$INSTDIR\uninstall.exe"
|
||||
|
||||
@@ -19,7 +19,7 @@ list="$target/uninstall-list.nsh"
|
||||
setup="$out/Jarvis-Setup-$version.exe"
|
||||
# The uninstall entry's size: what this setup installs, counted here. Counting the install folder on
|
||||
# the user's PC would also walk a data folder kept inside it (models, runtime: many thousand files).
|
||||
size_kb=$(cd "$out/Jarvis" && du -sk --apparent-size -c jarvis.exe WebView2Loader.dll python-src display-driver | tail -1 | cut -f1)
|
||||
size_kb=$(cd "$out/Jarvis" && du -sk --apparent-size -c jarvis.exe WebView2Loader.dll agent-desktop.exe python-src display-driver | tail -1 | cut -f1)
|
||||
# The shortcuts' icon file is named by its content, so a new icon gets a path Windows has not seen.
|
||||
icon_file="jarvis-$(sha256sum "$here/icons/icon.ico" | cut -c1-12).ico"
|
||||
makensis -V2 -DVERSION="$version" -DSRC="$out/Jarvis" -DOUT="$setup" -DICON="$here/icons/icon.ico" -DICON_FILE="$icon_file" \
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
#!/bin/sh
|
||||
# Build the Windows management app from Linux and lay it out ready to zip (admin.spec.md, "Windows build"):
|
||||
# Jarvis/jarvis.exe, Jarvis/WebView2Loader.dll, Jarvis/python-src/{voice_worker,jarvis},
|
||||
# Jarvis/display-driver/ (Virtual Display Driver 25.7.23, MIT, SignPath-signed; "Virtual monitor")
|
||||
# Needs the x86_64-pc-windows-gnu Rust target, mingw-w64, cmake (for Opus), curl and unzip.
|
||||
# Jarvis/display-driver/ (Virtual Display Driver 25.7.23, MIT, SignPath-signed; "Virtual monitor"),
|
||||
# Jarvis/agent-desktop.exe (agent-desktop 1.2.0, Apache-2.0; "PC control")
|
||||
# Needs the x86_64-pc-windows-gnu Rust target, mingw-w64, cmake (for Opus), curl, unzip and tar.
|
||||
# app/shell/package.sh <output folder>
|
||||
set -eu
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
@@ -30,4 +31,14 @@ unzip -q "$vdd_zip" -d "$tmp"
|
||||
mkdir -p "$out/Jarvis/display-driver"
|
||||
cp "$tmp/VirtualDisplayDriver/MttVDD.inf" "$tmp/VirtualDisplayDriver/mttvdd.cat" "$tmp/VirtualDisplayDriver/MttVDD.dll" "$out/Jarvis/display-driver/"
|
||||
rm -rf "$tmp"
|
||||
# PC control's agent-desktop, the archive and the program pinned by hash as in app/src/pc.rs (the
|
||||
# app checks the program's hash again before every start of the services).
|
||||
ad_url=https://github.com/vercel-labs/agent-desktop/releases/download/v1.2.0/agent-desktop-v1.2.0-x86_64-pc-windows-msvc.tar.gz
|
||||
ad_sha=a4199f741a441cedd349bf56c5eba01783b1261258b498efeb7ec9945220c3e8
|
||||
ad_exe_sha=d274667031322e50b9c2b1ab81c2ca46835d52cc932769a5c34bd4d481fd94c3
|
||||
ad_tgz="$target/agent-desktop-1.2.0.tar.gz"
|
||||
[ -f "$ad_tgz" ] || curl -fsSL -o "$ad_tgz" "$ad_url"
|
||||
echo "$ad_sha $ad_tgz" | sha256sum -c - >/dev/null || { rm -f "$ad_tgz"; echo "agent-desktop hash mismatch" >&2; exit 1; }
|
||||
tar -xzf "$ad_tgz" -C "$out/Jarvis" agent-desktop.exe
|
||||
echo "$ad_exe_sha $out/Jarvis/agent-desktop.exe" | sha256sum -c - >/dev/null || { echo "agent-desktop.exe hash mismatch" >&2; exit 1; }
|
||||
echo "$out/Jarvis"
|
||||
|
||||
@@ -66,9 +66,10 @@ manages the MCP servers it connects.
|
||||
"Virtual monitor"), off by default; `schedule` (reminders and scheduled tasks, below
|
||||
"Schedules", and the brain tool `schedule`), on by default; `notes` (personal notes, below
|
||||
"Personal notes", and the brain tool `personalNotes`), on by default; `screen` (화면 보기, below
|
||||
"Screen viewing", and the brain tool `screenView`), on by default. The tools of a feature that
|
||||
is off go into the brain's `disabled_tools`: they are neither offered nor run
|
||||
(`voice_worker.spec.md`, "Tools"). PC control comes with its milestone (M7).
|
||||
"Screen viewing", and the brain tool `screenView`), on by default; `pcControl` (PC 제어, below "PC
|
||||
control", and the brain tool `pcControl`), on by default (like `screen`, nothing is done until
|
||||
someone is in `pcControl.allowedUsers`). The tools of a feature that is off go into the brain's
|
||||
`disabled_tools`: they are neither offered nor run (`voice_worker.spec.md`, "Tools").
|
||||
- `mcpServers` (new list): each `{name, command, args, env, enabled}`. `name` is 1-32 letters,
|
||||
digits, `-` or `_` without `__` (the brain names a server's tools `<name>__<tool>`), not listed
|
||||
twice; `command` is not empty; `args` a list of strings; `env` the names of the environment
|
||||
@@ -166,8 +167,12 @@ decision 2026-11-14). What Jarvis may do on this PC and for whom; the PC control
|
||||
agrees in their next message (checked by the PC control tool like the chat delete consent).
|
||||
- Never done, whatever the settings: payments, passwords and other secrets, deleting files, and
|
||||
anything that needs administrator rights. They have no setting.
|
||||
- The PC 제어 screen edits the section and shows, read-only, what is never done and each registered
|
||||
chat channel's send and delete switches (changed on the 채팅 채널 screen).
|
||||
- `antiCheatGames` (program file names, ".exe" may be left out, at most 100, no folders; empty by
|
||||
default) and `stopHotkey` (modifiers `Ctrl`, `Alt`, `Shift`, `Win` and one letter, digit or
|
||||
F1-F24, at least one modifier; default `Ctrl+Shift+F12`): below "PC control".
|
||||
- The PC 제어 screen edits the section and shows the emergency stop's state with 풀기 (below "PC
|
||||
control"), and, read-only, what is never done and each registered chat channel's send and
|
||||
delete switches (changed on the 채팅 채널 screen).
|
||||
|
||||
## Screen viewing (`app/src/screen.rs`)
|
||||
|
||||
@@ -189,6 +194,21 @@ one window; the app captures and decides who may.
|
||||
`window` takes the visible top-level window whose title or program name contains the text
|
||||
(case-insensitive; the one nearest the front when several do), also when others cover it; a
|
||||
minimised window is refused ("최소화되어 있어 볼 수 없습니다").
|
||||
- **Screen share** (`share`, `docs/v2/pc-features.md` "다른 사람의 디스코드 화면공유 보기"):
|
||||
someone's Discord screen share, seen in the window Discord pops it out to; no account or token is
|
||||
touched and Jarvis never clicks Discord. Only while Jarvis is in a voice channel ("자비스가 음성
|
||||
채널에 있을 때만 화면공유를 볼 수 있습니다") and someone there other than Jarvis shares their
|
||||
screen (`self_stream`): with a `person`, the sharer whose display name contains it or is contained
|
||||
in it (letters and digits, any case), else "<person>님은 지금 화면공유 중이 아닙니다"; with none,
|
||||
anyone sharing ("지금 음성 채널에서 화면공유 중인 사람이 없습니다" when nobody does). The window
|
||||
is a visible Discord window (`discord.exe`, `discordptb.exe`, `discordcanary.exe`) whose title does
|
||||
not end with "Discord" (the main window's does), the one whose title contains the voice channel's
|
||||
name first, else the one nearest the front; Discord on the exclusion list is never seen. A
|
||||
minimised one is refused ("창을 다시 열어 주세요"). With none, a voice request says "디스코드에서
|
||||
<name>님 화면공유를 시청하고 창 분리 버튼을 눌러 주세요." in the channel (without a name when
|
||||
several share) and looks again every second for 30 s, then refuses ("화면공유 창이 분리되지 않아
|
||||
볼 수 없습니다"); a chat request is refused at once with that instruction. Who may see is decided
|
||||
before anything is said. The worker waits 45 s for a screen action ("PC control" timing).
|
||||
- **Exclusions**: `pcControl.screenExclude` (program file names, ".exe" may be left out; default
|
||||
`1Password`, `Bitwarden`, `KeePass`, `KeePassXC`). Their windows are never captured or listed, and
|
||||
in a monitor or whole-screen picture their area is painted black before anything leaves the
|
||||
@@ -207,8 +227,8 @@ one window; the app captures and decides who may.
|
||||
saved, never logged.
|
||||
- **List**: monitors `{number, width, height, main}` and visible windows `{program, title,
|
||||
minimised}`, front first, at most 30, without excluded programs and the app's own windows.
|
||||
- **Activity**: each capture is `screen.view` with the target (`전체 화면`, `모니터 <n>`, `창`; never
|
||||
a title), who asked and whether it worked ("Activity").
|
||||
- **Activity**: each capture is `screen.view` with the target (`전체 화면`, `모니터 <n>`, `창`,
|
||||
`화면공유`; never a title or a name), who asked and whether it worked ("Activity").
|
||||
- Logs carry the action, target kind, size and time taken, never a title or a picture.
|
||||
|
||||
## PC control (`app/src/pc.rs`)
|
||||
@@ -220,10 +240,65 @@ agent-desktop and decides who may.
|
||||
- **agent-desktop**: version 1.2.0 (Apache-2.0), pinned with its SHA-256, shipped in the installer
|
||||
next to `jarvis.exe` (`agent-desktop.exe`, about 4 MB) and checked before every start of the
|
||||
services (a wrong hash means not available). One CLI run per command (M0 `desktop`: 0.10 s, as
|
||||
fast as its MCP server), its APPDATA and LOCALAPPDATA in the data folder (`runtime/agent-desktop`),
|
||||
a 15 s limit per command. Actions are headless (no cursor, no focus), except `keys`, which brings
|
||||
the window to the front first. `--force` (which lifts its block on Alt+F4, Win+R, Ctrl+Shift+Esc
|
||||
and the like) and the raw mouse and key commands are never used.
|
||||
fast as its MCP server), its APPDATA, LOCALAPPDATA and state folder (`AGENT_DESKTOP_HOME`, where
|
||||
it keeps its readings) in the data folder (`runtime/agent-desktop`, made afresh at each start and
|
||||
private to the person: agent-desktop refuses a state folder others may write to, which a folder
|
||||
outside the user's profile is by inheritance), a 15 s limit per command. A
|
||||
program it starts gets the person's own APPDATA and LOCALAPPDATA back and starts in their user
|
||||
folder. Actions are headless (no cursor, no focus), `keys` as below. `--force`
|
||||
(which lifts its block on Alt+F4, Win+R, Ctrl+Shift+Esc and the like) and the raw mouse and key
|
||||
commands are never used; what the brain sends is checked so nothing in it reads as an option,
|
||||
and typed text goes in through stdin (`set-value --stdin`), never in the arguments. The commands: `list-windows`, `snapshot
|
||||
--window-id` (`read`), `click`, `get --property states` then `set-value` (`type`), `focus-window`
|
||||
then `press` (`keys`), `launch` (`run`), `close-app` (`close`: agent-desktop has no command for
|
||||
one window, so the program closes gracefully with all its windows; one still running when
|
||||
agent-desktop gives up, a save question say, is "프로그램이 아직 닫히지 않았습니다(저장할지 묻는 창이
|
||||
떠 있을 수 있습니다). 창을 다시 읽어 주세요", and with several of it running close-app cannot
|
||||
choose: "그 프로그램이 여러 개 실행 중이라 이 창만 닫을 수 없습니다. 창의 닫기 단추를 눌러 주세요"),
|
||||
`restore`.
|
||||
- **Keys** (`keys`; user decision 2026-11-19): one way is chosen before any key is sent, from the
|
||||
keys and the window, and never tried the other way after (a key is never pressed twice):
|
||||
- **Behind** (the default): one key without Ctrl, Alt, Shift or Win that types or moves (`a`-`z`,
|
||||
`0`-`9`, `enter`, `tab`, `space`, `escape`, `delete`/`backspace`, `forwarddelete`, the arrows,
|
||||
`home`, `end`, `pageup`, `pagedown`, `f1`-`f9`, `f11`, `f12`; agent-desktop's names) is posted to
|
||||
the window's text control (`PostMessage`, key down then key up; the program's own message
|
||||
loop makes the character) without bringing the window forward: the person's screen and focus
|
||||
stay as they are. Only for a window with exactly one text control shown (`Edit`, `RichEdit20A`,
|
||||
`RichEdit20W`, `RICHEDIT50W`, the new Notepad's `RichEditD2DPT`, `Scintilla`), which gets the
|
||||
keys: Windows does not tell which control of a window behind has the focus (a thread behind
|
||||
has none; win-ci VM 2026-11-19), so keys go behind only where there is no doubt where they land.
|
||||
Store and WinUI apps (the new Calculator), Chromium, Electron and Firefox windows have none.
|
||||
- **In front for a moment**: a combination, any other key, and every key for a window that does
|
||||
not take keys behind or is minimised. A minimised window is restored inside the moment
|
||||
(restoring brings it forward, so the window in front is noted before and brought back after;
|
||||
the window itself stays restored). The window is brought to the front (`focus-window`, then `press`) and the
|
||||
window that was in front before is brought back right after (`focus-window` on it, also when
|
||||
`press` failed; one that cannot be brought back is left, with a warning in the log). For a
|
||||
window whose process agent-desktop cannot find from the window (the new Calculator, a Store
|
||||
app: `press --window-id` is `APP_NOT_FOUND` and sends nothing; game PC 2026-11-19) the keys
|
||||
are pressed without naming the window (nothing was sent before), into the window in front,
|
||||
only while that is still the window (checked just before). A key the
|
||||
person types in that moment may go to the window. When Windows does not let the window come
|
||||
forward (its foreground lock), no key is sent ("그 창을 앞으로 가져오지 못해 키를 보내지
|
||||
않았습니다"). A window already in front gets the keys there and nothing else moves.
|
||||
- **Never in front during a game or full screen**: while game mode is on, the window in front
|
||||
covers its whole monitor without a title bar or sizing border (a video or a program full
|
||||
screen; not the desktop), or Windows reports a full-screen Direct3D program, keys that need
|
||||
the window in front are not sent ("게임 중이거나 전체 화면이라 그 창을 앞으로 가져오지 않았습니다.
|
||||
키를 보내지 않았습니다").
|
||||
- Either way the window is read again after ("Results"); keys a program did not take behind show
|
||||
there as no change, and the brain does not say they were pressed.
|
||||
- **Windows**: as `list-windows` gives them, without this app's own windows, windows without a
|
||||
title and the programs of `pcControl.screenExclude` (never listed, read or acted on, as for
|
||||
screen viewing); at most 30. `window` is a window id or part of a title or program name (the
|
||||
focused match first, else the first listed). An element (`@<snapshot>:e<n>`) acts on the window
|
||||
it was read from; one from a reading this run did not make gets `STALE_REF`'s reason. `run` takes
|
||||
a program name only (no folders): one in Windows' folders as it is, else the file Windows' App
|
||||
Paths or a Start menu shortcut without arguments points at; the window is the one `launch` names,
|
||||
else a new one of its process or name, else the only new one, waited for up to 5 s. A program
|
||||
already running is not started again: `launch` attaches to it and names its window; with several
|
||||
running it cannot choose ("그 프로그램이 이미 여러 개 실행 중이라 새로 열지 못했습니다. 열린 창 중에서
|
||||
골라 주세요").
|
||||
- **Available** only on Windows with `features.pcControl` on, someone in `pcControl.allowedUsers`
|
||||
and agent-desktop present; otherwise `pcControl` is in the brain's `disabled_tools`. Settings
|
||||
apply on the next start.
|
||||
@@ -233,8 +308,8 @@ agent-desktop and decides who may.
|
||||
사람만 PC를 조작할 수 있습니다"), a kind at `never` ("PC 제어 화면에서 이 동작이 꺼져 있습니다").
|
||||
At `ask` the action is done only with `consent: true`, else the answer is `ask: true` with the
|
||||
kinds and the window (the brain asks the person; the app does not keep consent).
|
||||
- **Never**: typing into a password field (agent-desktop reports it, or Windows UI Automation's
|
||||
IsPassword) is refused ("비밀번호 칸에는 입력하지 않습니다"); a window run as administrator is
|
||||
- **Never**: typing into a password field (agent-desktop reports it as `secure`: in the reading the
|
||||
element came from, anywhere in its tree, or by `get` now) is refused ("비밀번호 칸에는 입력하지 않습니다"); a window run as administrator is
|
||||
refused before acting ("관리자 권한으로 실행된 창은 조작할 수 없습니다"), and `run` never asks for
|
||||
administrator rights.
|
||||
- **Anti-cheat**: while a game with an anti-cheat runs, every action but `windows` and `read` is
|
||||
@@ -243,20 +318,30 @@ agent-desktop and decides who may.
|
||||
(VALORANT, League of Legends; the driver is always loaded), Easy Anti-Cheat, BattlEye, FACEIT,
|
||||
nProtect GameGuard and XIGNCODE3 by their own processes (list in `pc.rs`, grown by updates), and
|
||||
`pcControl.antiCheatGames` (program file names the person added on the PC 제어 screen; no switch
|
||||
turns the check off). The game's name is the game detection's when it runs, else the program's.
|
||||
turns the check off). The game's name is the game detection's when it runs, else the program's
|
||||
(Riot's processes are named as their games). Processes are found by their file names in Windows'
|
||||
process list, also those this user may not open.
|
||||
- **Emergency stop**: a global hotkey (`pcControl.stopHotkey`, default `Ctrl+Shift+F12`) stops at
|
||||
once: the running agent-desktop command is killed and every action but `windows` and `read` is
|
||||
refused ("비상 정지 중입니다. 단축키를 한 번 더 누르거나 PC 제어 화면에서 풀어 주세요") until the
|
||||
hotkey is pressed again or the PC 제어 screen lifts it. The dashboard shows "PC 제어: 비상 정지";
|
||||
an app restart lifts it.
|
||||
an app restart lifts it. The hotkey is registered (`RegisterHotKey`, its own thread and message
|
||||
loop) while the services run with PC control available; one Windows refuses (another program
|
||||
holds it) is a warning in the log. A reading started while the stop is on runs to its end.
|
||||
- **Results**: `read` and every action answer with the window `{id, program, title}` and its
|
||||
interactive elements read afresh after the action (at most 200), so the brain says it was done
|
||||
only from what it sees. agent-desktop's error codes come back as Korean reasons (`STALE_REF`: "화면이
|
||||
바뀌었습니다. 다시 읽어 주세요"; a minimised window is restored before acting).
|
||||
only from what it sees; an action that closes its window (a dialog's button, Enter) answers the
|
||||
window with no elements. agent-desktop's error codes come back as Korean reasons (`STALE_REF`: "화면이
|
||||
바뀌었습니다. 다시 읽어 주세요"; a minimised window is restored before acting; for `keys` inside
|
||||
the moment in front, "Keys").
|
||||
- **Refused before asking**: the stop, the anti-cheat, an administrator window (a process whose
|
||||
elevation cannot be read counts as one) and a password field are refused without asking.
|
||||
`windows` and `read` at `ask` answer `ask: true` like the rest.
|
||||
- **Activity**: each action but `windows` and `read` is `pc.<action>` with the program's file name
|
||||
(never a title, element name or text), who asked and whether it worked ("Activity").
|
||||
- Logs carry the action, the result code and the time taken; never a title, element name, value or
|
||||
typed text.
|
||||
(never a title, element name or text), who asked and whether it worked, with the refusal's reason
|
||||
("Activity"); an `ask` answer is not recorded (nothing was done).
|
||||
- Logs carry the action, the result code and the time taken (for `keys` also the way: behind or in
|
||||
front); never a title, element name, value, typed text or key.
|
||||
|
||||
## Personal notes (`app/src/notes.rs`)
|
||||
|
||||
@@ -309,7 +394,7 @@ records and "Traces"), so it works while the services are stopped. Nothing it re
|
||||
`docs/v2/proactive.md` (제안 5). What Jarvis did outside its answers, for the 로그 screen's 한 일 view: each chat
|
||||
channel action the brain's tool asked for (`chat.send`, `chat.edit`, `chat.react`, `chat.delete`,
|
||||
`chat.pin`, `chat.unpin`), each schedule delivery (`schedule.remind`, `schedule.task`) and each
|
||||
news notification (`collect.notify`), each screen capture (`screen.view`, "Screen viewing"), done or failed. The other PC control actions join it with M7. Ordinary answers are not recorded.
|
||||
news notification (`collect.notify`), each screen capture (`screen.view`, "Screen viewing") and each PC control action (`pc.click`, `pc.type`, `pc.keys`, `pc.run`, `pc.close`, "PC control"), done or failed. Ordinary answers are not recorded.
|
||||
|
||||
- One JSON line per action in `activity/<YYYY-MM-DD>.jsonl` (the folder is made at the first one):
|
||||
`{at, kind, target, by, ok, note}`. `target` is the channel (and message) id or `예약 <id>`; `by`
|
||||
@@ -513,6 +598,10 @@ background (headless), or with `features.browserShare` in a window on the virtua
|
||||
- Video and sound start when a page starts them (`--autoplay-policy=no-user-gesture-required`): the
|
||||
botBrowser tool clicks with page scripts, which Chrome does not count as a user's gesture, so a
|
||||
video page opened for "play X" stayed paused (live, 2026-11-16).
|
||||
- Pages are not told a program drives the browser (`--disable-blink-features=AutomationControlled`):
|
||||
DevTools turns `navigator.webdriver` on, and with it YouTube stopped every video about 45 s in
|
||||
with "문제가 발생했습니다" (`ump.spsrejectfailure`, live 2026-11-18, 0.1.40; the M0 video check had
|
||||
the same on 2026-11-11).
|
||||
- **Egress proxy**: every Chrome connection goes through the app's proxy on a free loopback port
|
||||
(`--proxy-server`, and `--proxy-bypass-list=<-loopback>` so not even loopback is direct; QUIC off,
|
||||
WebRTC only through the proxy). The proxy takes `CONNECT host:port` (HTTPS, WebSocket) and plain
|
||||
@@ -772,6 +861,7 @@ back as Korean messages for the screen. The Tauri shell passes the screens' `inv
|
||||
| `settings_fields` / `settings_view` / `settings_apply` `{changes}` | the settings editor above; `restartNeeded` when the services run and something other than `voice.tts.speed` changed (the speed reaches the running worker at once, `discord_core.spec.md`); `sync` with the chat channels sent to Discord (below) |
|
||||
| `service_start` / `service_stop` / `service_restart` / `dashboard` | the controller and its snapshot; start and restart run in the background, stop also cancels a start |
|
||||
| `discord_channel_info` `{id}` / `channel_sync_status` | a channel's Discord name and topic / each synced chat channel's last result (above) |
|
||||
| `pc_stop_status` / `pc_stop_lift` | `{stopped}`: PC control's emergency stop (above, "PC control") / lift it |
|
||||
| `notes_list` / `note_update` `{id, text}` / `note_delete` `{id}` | everyone's personal notes (above) / change one's text / delete one |
|
||||
| `history_query` `{said, answered, thought, via, person, channel, find, since, until, session, before, from, limit}` | the 대화 screen's records and answer traces (above, "Conversations") |
|
||||
| `history_stamp` | `{stamp}`: changes when what the 대화 screen shows may have changed (above, "Conversations") |
|
||||
@@ -795,8 +885,13 @@ its data folder, and again after each one is saved. The log hub writes to the da
|
||||
## Screens (`app/ui/`)
|
||||
|
||||
Plain HTML, CSS and an ES module, no build step, dark theme after `DESIGN.md`. All text is Korean.
|
||||
The side menu has 대화 right under 대시보드 and 앱 last; there is no separate update screen (updates
|
||||
are on the 앱 screen) and no separate 한 일 screen (it is a view of the 로그 screen).
|
||||
The side menu has 대시보드 and 대화 on top, then groups that open and close when their name is
|
||||
pressed (user request, 2026-11-19): 디스코드 (봇, 음성 채널, 채팅 채널), 대화 방식 (호출, 여럿이 말할
|
||||
때, 대화 기억, 사람별 지침), 음성·모델 (받아 적기·말하기, 대화 모델), 기능 (기능 켜기·MCP, 예약,
|
||||
백그라운드 수집), PC (PC 제어·화면 보기, 게임 모드) and 관리 (로그, 앱). The group of the screen
|
||||
shown opens (and the 관리 group when a new version renames the 앱 button); which groups are open is
|
||||
kept in the page's storage for the next start. There is no separate update screen (updates are on
|
||||
the 앱 screen) and no separate 한 일 screen (it is a view of the 로그 screen).
|
||||
|
||||
- **First-run wizard** (while `firstRun`): data folder (free space, warning under 200 GB), PC check,
|
||||
the model to download, Discord bot (token check with the
|
||||
@@ -809,7 +904,7 @@ are on the 앱 screen) and no separate 한 일 screen (it is a view of the 로
|
||||
Strata's last line, and stop in the header; the header with start, restart and stop is rebuilt
|
||||
only when the state changes, so a click is never lost to the refresh. Every screen change starts
|
||||
a new generation: an answer that arrives after the user moved to another screen is dropped.
|
||||
- **Settings screens** (봇, 채널·호출, 채팅 채널, 대화 기억, 모델, 기능, 백그라운드 수집, 예약, PC 제어, 게임 모드, 앱): built from `settings_fields`; only
|
||||
- **Settings screens** (봇, 음성 채널, 호출, 여럿이 말할 때, 채팅 채널, 대화 기억, 받아 적기·말하기, 대화 모델, 기능 켜기·MCP, 백그라운드 수집, 예약, PC 제어·화면 보기, 게임 모드, 앱): built from `settings_fields`, each field on one screen: 음성 채널 has the server, the auto-join channel, the speakers, bots, the join notice and the recording switch; 호출 the voice and chat wake words and `call.*`; 여럿이 말할 때 `response.*`; 받아 적기·말하기 `voice.stt.*` and the speech speed; 대화 모델 `llm.*`; the default location is on 기능 켜기·MCP (weather and search use it); only
|
||||
changed fields are sent; the server and voice channel are pickers (choosing another server clears
|
||||
the voice channel and lists only that server's channels); the chat channels are a table
|
||||
with their switches; the bot screen also changes the token; the 기능 screen has the feature
|
||||
@@ -920,7 +1015,8 @@ both to a release with that tag. The repository's other releases (v1's `v1.x`) a
|
||||
`icons/icon.png` 512 px): voice bars in a ring on the accent colour's gradient; at 32 px and below
|
||||
only three thicker bars, so it stays readable in the taskbar and tray.
|
||||
- `app/shell/package.sh <folder>` builds it on Linux (`x86_64-pc-windows-gnu`, mingw-w64) and lays
|
||||
out `Jarvis/`: `jarvis.exe`, `WebView2Loader.dll`, `python-src/` (the `voice_worker` and
|
||||
out `Jarvis/`: `jarvis.exe`, `WebView2Loader.dll`, `agent-desktop.exe` (PC control, from the
|
||||
pinned release archive, both hashes checked), `python-src/` (the `voice_worker` and
|
||||
`jarvis` packages, no caches or specs) and `display-driver/` (the virtual monitor's driver). The worker's sources are `python-src` next to the
|
||||
executable when it is there, also over a `voiceWorker.cwd` the settings keep (another install's
|
||||
first-run setup may have written its own folder there, and an update must never run those older
|
||||
@@ -1042,14 +1138,30 @@ a path with non-ASCII letters) or from a Battle.net game's install folder (a tes
|
||||
turns game mode on with its name and off once it ends. Screen viewing (`tests/screen_windows.rs`,
|
||||
`JARVIS_TEST_SCREEN=1`): the monitors and Notepad's window are listed, the window, each monitor and
|
||||
the whole screen come back as JPEG pictures of their size, and with Notepad excluded it is neither
|
||||
picked nor listed and its area is black on the whole screen; someone not registered is refused. On
|
||||
picked nor listed and its area is black on the whole screen; someone not registered is refused.
|
||||
Keys (`tests/pc_keys_windows.rs`, `JARVIS_TEST_PC_KEYS=1`, agent-desktop at `JARVIS_AGENT_DESKTOP`;
|
||||
it opens Notepad, Character Map and the Calculator, and refuses to start while any of them runs):
|
||||
with Character Map in front, letters and Enter reach Notepad behind it and Character Map stays in
|
||||
front; a combination and the Calculator's keys go in front for a moment and Character Map comes
|
||||
back. On
|
||||
every system with a fake screen: refusals before any capture, monitors by number, the whole screen
|
||||
as placed with excluded areas black (also at another display scale), an excluded window that
|
||||
moves or is brought back during the capture never shown (taken again, or refused while it keeps
|
||||
moving), a window of a program that cannot be told treated as excluded, windows by title or program
|
||||
front first (never excluded, secondary or the app's own), minimised and black pictures refused, the
|
||||
detail sizes and parts; the worker protocol test carries a 4K capture's pictures to a fake brain and
|
||||
back. Performance stages (on every system, with a fake
|
||||
back. PC control (on every system, with a fake agent-desktop): people not registered and kinds at
|
||||
`never` refused before any command, `ask` answered without consent and done with it, the
|
||||
anti-cheat refusing every action but reading (its own processes and the person's list, named by
|
||||
game detection), the emergency stop refusing and lifted, and the real runner killing a running
|
||||
command when it is pressed and at its time limit, a password field and an administrator window
|
||||
refused before acting, readings capped at 200 elements (interactive first) and cut to 120
|
||||
characters, `STALE_REF`'s reason, a minimised window restored, `keys` sent behind or in front by the keys and the window (one way
|
||||
only, the window in front brought back, also after a failed press) and never in front during a
|
||||
game or full screen,
|
||||
excluded and own windows never listed, `run` reading its new window, the activity naming the
|
||||
program only, the hotkey and shortcut formats, and one pin of agent-desktop in `pc.rs`,
|
||||
`package.sh` and M0. Performance stages (on every system, with a fake
|
||||
Strata and card): a game moves the LLM to the game placement (measured once at 16K and one request,
|
||||
2 GiB on the game PC's numbers, half the 16 engine threads) and back after the wait, pause stops
|
||||
it, a card with no room for a game pauses, the normal stage counts a program opened since, the game
|
||||
|
||||
@@ -410,6 +410,12 @@ impl AdminApi {
|
||||
v
|
||||
}).collect::<Vec<_>>()}))
|
||||
}
|
||||
// PC control's emergency stop (admin.spec.md, "PC control"): its state, and 풀기.
|
||||
"pc_stop_status" => Ok(json!({"stopped": crate::pc::emergency().on()})),
|
||||
"pc_stop_lift" => {
|
||||
crate::pc::emergency().lift();
|
||||
Ok(json!({"stopped": false}))
|
||||
}
|
||||
// Personal notes (admin.spec.md, "Personal notes"): everyone's, editable on the screen.
|
||||
"notes_list" => {
|
||||
let items = crate::notes::Notes::new(self.dir()?.path()).all()?;
|
||||
|
||||
@@ -237,6 +237,10 @@ pub fn chrome_args(profile: &Path, proxy_port: u16, no_sandbox: bool, screen: bo
|
||||
// Video and sound start when a page starts them: the tool clicks with page scripts, which
|
||||
// Chrome does not count as a user's gesture, so a video page opened for "play X" stayed paused.
|
||||
"--autoplay-policy=no-user-gesture-required".into(),
|
||||
// Pages are not told a program drives the browser (navigator.webdriver, which DevTools
|
||||
// turns on): with it YouTube stopped every video about 45 s in with "문제가 발생했습니다"
|
||||
// (ump.spsrejectfailure, live 2026-11-18; the M0 video check had the same, 2026-11-11).
|
||||
"--disable-blink-features=AutomationControlled".into(),
|
||||
];
|
||||
if no_sandbox {
|
||||
a.push("--no-sandbox".into());
|
||||
|
||||
@@ -31,6 +31,13 @@ pub struct CallSettings {
|
||||
/// After a wake word said alone, how long the speaker's next utterance may take to start and
|
||||
/// still be joined to it (the pause after "Jarvis," often ends an utterance).
|
||||
pub follow_up: Duration,
|
||||
/// A speaker who goes on talking before Jarvis's answer to them is heard, starting within this of
|
||||
/// their last utterance's end, is still asking: what they say is joined to the request
|
||||
/// (`call.continueMs`; zero turns it off).
|
||||
pub continue_window: Duration,
|
||||
/// A voice answer starts playing only once its asker has been quiet this long after their request
|
||||
/// ended (they may go on); it is written meanwhile (`call.answerWaitMs`; zero: as soon as written).
|
||||
pub answer_wait: Duration,
|
||||
/// Play the earcon when Jarvis takes a voice call (`call.earcon`).
|
||||
pub earcon: bool,
|
||||
}
|
||||
@@ -38,7 +45,7 @@ pub struct CallSettings {
|
||||
impl Default for CallSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
wake_words: vec!["자비스".into()],
|
||||
wake_words: crate::settings::default_wake_words(),
|
||||
similarity: 0.2,
|
||||
mode: CallMode::WakeWord,
|
||||
designated: HashSet::new(),
|
||||
@@ -46,6 +53,8 @@ impl Default for CallSettings {
|
||||
designated_no_wake_word: false,
|
||||
stop_words: ["멈춰", "그만", "정지", "stop"].map(String::from).to_vec(),
|
||||
follow_up: Duration::from_millis(1500),
|
||||
continue_window: Duration::from_millis(5000),
|
||||
answer_wait: Duration::from_millis(3000),
|
||||
earcon: true,
|
||||
}
|
||||
}
|
||||
@@ -70,9 +79,9 @@ pub fn normalise(text: &str) -> Vec<char> {
|
||||
text.nfkd().flat_map(char::to_lowercase).filter(|c| c.is_alphanumeric()).collect()
|
||||
}
|
||||
|
||||
/// The stretch `[start, end)` of `text` closest to `word`, if within `max_edits` edits
|
||||
/// (approximate substring matching: the match may start and end anywhere in `text`).
|
||||
fn find(text: &[char], word: &[char], max_edits: usize) -> Option<(usize, usize)> {
|
||||
/// The stretch `[start, end)` of `text` closest to `word` and its edits, if within `max_edits`
|
||||
/// edits (approximate substring matching: the match may start and end anywhere in `text`).
|
||||
fn find(text: &[char], word: &[char], max_edits: usize) -> Option<(usize, usize, usize)> {
|
||||
if word.is_empty() || text.is_empty() {
|
||||
return None;
|
||||
}
|
||||
@@ -104,7 +113,7 @@ fn find(text: &[char], word: &[char], max_edits: usize) -> Option<(usize, usize)
|
||||
(1..=n)
|
||||
.filter(|&j| d[m][j] <= max_edits)
|
||||
.min_by_key(|&j| (d[m][j], j))
|
||||
.map(|j| (start[m][j], j))
|
||||
.map(|j| (start[m][j], j, d[m][j]))
|
||||
}
|
||||
|
||||
impl CallSettings {
|
||||
@@ -114,13 +123,15 @@ impl CallSettings {
|
||||
|
||||
/// The text with the first heard wake word removed, if one is heard.
|
||||
fn without_wake_word(&self, text: &[char]) -> Option<Vec<char>> {
|
||||
// The earliest heard wake word is the one used to address Jarvis.
|
||||
let (s, e) = self
|
||||
// The earliest heard wake word is the one used to address Jarvis; where several start
|
||||
// there, the closest, then the longest ("비서야" over "비서"), so no part of the call is
|
||||
// left over and a near match does not eat into what follows.
|
||||
let (s, e, _) = self
|
||||
.wake_words
|
||||
.iter()
|
||||
.map(|w| normalise(w))
|
||||
.filter_map(|word| find(text, &word, self.allowed_edits(&word)))
|
||||
.min()?;
|
||||
.min_by_key(|&(s, e, edits)| (s, edits, std::cmp::Reverse(e)))?;
|
||||
Some(text[..s].iter().chain(&text[e..]).copied().collect())
|
||||
}
|
||||
|
||||
@@ -169,8 +180,9 @@ mod tests {
|
||||
const A: u64 = 1;
|
||||
const B: u64 = 2;
|
||||
|
||||
/// The mechanism tests use one wake word of their own, whatever the default is.
|
||||
fn s() -> CallSettings {
|
||||
CallSettings::default()
|
||||
CallSettings { wake_words: vec!["자비스".into()], ..CallSettings::default() }
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -264,4 +276,25 @@ mod tests {
|
||||
assert_eq!(c.judge(A, "javis 날씨"), Call::Request { direct: true });
|
||||
assert_eq!(c.judge(A, "javis stop"), Call::Stop);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_longest_wake_word_where_the_call_starts_is_the_one_removed() {
|
||||
let c = CallSettings { wake_words: vec!["비서".into(), "비서야".into(), "김비서".into()], ..s() };
|
||||
assert!(c.wake_word_only("비서야,"), "the call form is the whole wake word, not 비서 and a 야 left over");
|
||||
assert!(c.wake_word_only("김비서?"));
|
||||
assert_eq!(c.judge(A, "비서야 멈춰"), Call::Stop);
|
||||
assert_eq!(c.judge(A, "김비서 그만해"), Call::Stop);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn by_default_jarvis_is_called_like_a_secretary() {
|
||||
let c = CallSettings::default();
|
||||
for call in ["비서", "비서야", "비서님", "김비서", "김비서야", "김비서님"] {
|
||||
assert!(c.wake_word_only(&format!("{call},")), "{call}");
|
||||
assert_eq!(c.judge(A, &format!("{call} 오늘 날씨 어때?")), Call::Request { direct: true }, "{call}");
|
||||
assert_eq!(c.judge(A, &format!("{call} 멈춰")), Call::Stop, "{call}");
|
||||
}
|
||||
assert_eq!(c.judge(A, "오늘 날씨 어때?"), Call::Ignore);
|
||||
assert_eq!(c.judge(A, "자비스 날씨"), Call::Ignore, "the old name no longer calls");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -303,6 +303,7 @@ impl Controller {
|
||||
view["llm"] = self.host.llm();
|
||||
view["game"] = status.get("game").cloned().unwrap_or(Value::Null);
|
||||
view["performance"] = status.get("performance").cloned().unwrap_or(Value::Null);
|
||||
view["pcStop"] = json!(crate::pc::emergency().on());
|
||||
view
|
||||
}
|
||||
|
||||
|
||||
@@ -33,7 +33,10 @@ pub struct Room {
|
||||
pub trait Voice {
|
||||
/// Have the brain answer and speak it; the speech request id, or None when it cannot. `heard`:
|
||||
/// when the request's speech ended and its final arrived (the dashboard times the answer by it).
|
||||
fn answer(&self, user: u64, text: &str, heard: (Instant, Instant)) -> Option<String>;
|
||||
/// `hold`: the answer is written but does not start playing until `release`.
|
||||
fn answer(&self, user: u64, text: &str, heard: (Instant, Instant), hold: bool) -> Option<String>;
|
||||
/// Let a held answer play.
|
||||
fn release(&self, id: &str);
|
||||
/// Stop one speech request.
|
||||
fn cancel(&self, id: &str);
|
||||
/// Stop all speech at once.
|
||||
@@ -57,8 +60,11 @@ pub trait Voice {
|
||||
}
|
||||
|
||||
impl Voice for DiscordCore {
|
||||
fn answer(&self, user: u64, text: &str, heard: (Instant, Instant)) -> Option<String> {
|
||||
DiscordCore::answer(self, user, text, heard)
|
||||
fn answer(&self, user: u64, text: &str, heard: (Instant, Instant), hold: bool) -> Option<String> {
|
||||
DiscordCore::answer(self, user, text, heard, hold)
|
||||
}
|
||||
fn release(&self, id: &str) {
|
||||
DiscordCore::release(self, id)
|
||||
}
|
||||
fn cancel(&self, id: &str) {
|
||||
DiscordCore::cancel(self, id)
|
||||
@@ -167,6 +173,25 @@ pub struct Conversation<V: Voice> {
|
||||
wakes: HashMap<u64, (Request, Option<Instant>)>,
|
||||
/// Requests whose earcon already played (a wake word said alone was answered with it).
|
||||
chimed: HashSet<u64>,
|
||||
/// Each speaker's latest voice request whose answer has not been heard yet: what they say next
|
||||
/// may still be part of it ("Going on").
|
||||
turns: HashMap<u64, Turn>,
|
||||
/// Held answers (speech id): their asker, and when they will have been quiet for `call.answerWaitMs`.
|
||||
waiting: HashMap<String, (u64, Instant)>,
|
||||
}
|
||||
|
||||
/// A voice request whose answer is not heard yet.
|
||||
struct Turn {
|
||||
id: u64,
|
||||
text: String,
|
||||
ended_at: Instant,
|
||||
/// Still a "was it said to Jarvis?" question (its ask id), not yet a request of the policy.
|
||||
ask: Option<String>,
|
||||
/// The speaker started their next utterance within `call.continueMs` while the answer was not
|
||||
/// heard: the request stays open for that utterance's final, also if the answer starts meanwhile.
|
||||
held: bool,
|
||||
/// Its answer started playing (kept only while `held`).
|
||||
heard: bool,
|
||||
}
|
||||
|
||||
impl<V: Voice> Conversation<V> {
|
||||
@@ -185,7 +210,7 @@ impl<V: Voice> Conversation<V> {
|
||||
Self { voice, calls, policy: Policy::new(policy), next_id: 0, answers: HashMap::new(),
|
||||
by_speech: HashMap::new(), asks: HashMap::new(), missed: MissedLog::default(), chat: ChatCalls::default(),
|
||||
chat_answers: HashMap::new(), started: HashMap::new(), wakes: HashMap::new(),
|
||||
chimed: HashSet::new() }
|
||||
chimed: HashSet::new(), turns: HashMap::new(), waiting: HashMap::new() }
|
||||
}
|
||||
|
||||
/// Answer the registered chat channels too.
|
||||
@@ -225,6 +250,13 @@ impl<V: Voice> Conversation<V> {
|
||||
/// would be) lets the brain get ready while the speaker is still talking. Not while Jarvis is
|
||||
/// answering: the call may not be taken, and the brain is busy with the answer.
|
||||
fn on_partial(&mut self, speaker: u64, partial: &str) {
|
||||
// Going on with a request whose answer is not heard yet: it will be joined and taken.
|
||||
if let Some(t) = self.turns.get(&speaker) {
|
||||
if !self.calls.continue_window.is_zero() && !matches!(self.calls.judge(speaker, partial), Call::Stop) {
|
||||
self.voice.prepare(speaker, &format!("{} {}", t.text, partial));
|
||||
}
|
||||
return;
|
||||
}
|
||||
if !self.answers.is_empty() {
|
||||
return;
|
||||
}
|
||||
@@ -248,6 +280,24 @@ impl<V: Voice> Conversation<V> {
|
||||
}
|
||||
|
||||
pub fn on_event(&mut self, ev: &CoreEvent, now: Instant) {
|
||||
self.handle(ev, now);
|
||||
self.release_due(now);
|
||||
}
|
||||
|
||||
/// Held answers whose asker has been quiet for `call.answerWaitMs` start playing; while the asker
|
||||
/// talks, their answer waits for what they say (it may join the request).
|
||||
fn release_due(&mut self, now: Instant) {
|
||||
let due: Vec<String> = self.waiting.iter()
|
||||
.filter(|(_, (speaker, at))| now >= *at && !self.started.contains_key(speaker))
|
||||
.map(|(sid, _)| sid.clone()).collect();
|
||||
for sid in due {
|
||||
self.waiting.remove(&sid);
|
||||
debug_log!("conversation", "release speech={sid}");
|
||||
self.voice.release(&sid);
|
||||
}
|
||||
}
|
||||
|
||||
fn handle(&mut self, ev: &CoreEvent, now: Instant) {
|
||||
match ev {
|
||||
CoreEvent::SpeechStart(h) => {
|
||||
self.policy.utterance_open(h.speaker_id);
|
||||
@@ -255,6 +305,10 @@ impl<V: Voice> Conversation<V> {
|
||||
if let Some((_, deadline)) = self.wakes.get_mut(&h.speaker_id) {
|
||||
*deadline = None; // they go on: wait for this utterance's final
|
||||
}
|
||||
let window = self.calls.continue_window;
|
||||
if let Some(t) = self.turns.get_mut(&h.speaker_id) {
|
||||
t.held = !window.is_zero() && now <= t.ended_at + window;
|
||||
}
|
||||
}
|
||||
CoreEvent::Chat(m) => self.on_chat(m, now),
|
||||
CoreEvent::Partial(h, partial) => self.on_partial(h.speaker_id, partial),
|
||||
@@ -277,10 +331,15 @@ impl<V: Voice> Conversation<V> {
|
||||
self.started.remove(&speaker);
|
||||
}
|
||||
// What follows a wake word said alone is joined to it.
|
||||
let (text, chimed) = match self.wakes.remove(&speaker) {
|
||||
let (mut text, mut chimed) = match self.wakes.remove(&speaker) {
|
||||
Some((wake, _)) => (format!("{} {}", wake.text, f.text), self.chimed.remove(&wake.id)),
|
||||
None => (f.text.clone(), false),
|
||||
};
|
||||
// The speaker goes on before their answer was heard: still the same request.
|
||||
if let Some((earlier, earcon_played)) = self.going_on(speaker, f, now) {
|
||||
text = format!("{earlier} {text}");
|
||||
chimed |= earcon_played;
|
||||
}
|
||||
self.next_id += 1;
|
||||
if chimed {
|
||||
self.chimed.insert(self.next_id);
|
||||
@@ -316,21 +375,19 @@ impl<V: Voice> Conversation<V> {
|
||||
let a = self.policy.utterance_closed(speaker, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
Call::Request { direct } => {
|
||||
let a = self.policy.request(Request { direct, ..req }, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
Call::Request { direct } => self.voice_request(Request { direct, ..req }, now),
|
||||
Call::Ask => {
|
||||
let room = self.room(speaker);
|
||||
if room.people <= 1 {
|
||||
// Alone with Jarvis: whatever the speaker says is said to it.
|
||||
debug_log!("conversation", "addressed req={} alone with jarvis", req.id);
|
||||
let a = self.policy.request(req, now);
|
||||
self.apply(a, now);
|
||||
self.voice_request(req, now);
|
||||
return;
|
||||
}
|
||||
let id = format!("q{}", req.id);
|
||||
if self.voice.ask_addressed(&id, speaker, &text, room) {
|
||||
self.turns.insert(speaker, Turn { id: req.id, text: req.text.clone(),
|
||||
ended_at: req.ended_at, ask: Some(id.clone()), held: self.holds(speaker), heard: false });
|
||||
self.asks.insert(id, (req, now));
|
||||
} else {
|
||||
let a = self.policy.utterance_closed(speaker, now);
|
||||
@@ -344,12 +401,13 @@ impl<V: Voice> Conversation<V> {
|
||||
// An answer after the timeout counts as no, even before the timer ran.
|
||||
let yes = *addressed && now < asked + ADDRESSED_TIMEOUT;
|
||||
debug_log!("conversation", "addressed req={} yes={yes}", req.id);
|
||||
let a = if yes {
|
||||
self.policy.request(req, now)
|
||||
self.turns.retain(|_, t| t.id != req.id);
|
||||
if yes {
|
||||
self.voice_request(req, now);
|
||||
} else {
|
||||
self.policy.utterance_closed(req.speaker, now)
|
||||
};
|
||||
self.apply(a, now);
|
||||
let a = self.policy.utterance_closed(req.speaker, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
}
|
||||
}
|
||||
CoreEvent::Ranked { id, order } => {
|
||||
@@ -358,9 +416,23 @@ impl<V: Voice> Conversation<V> {
|
||||
let a = self.policy.ranked(rank_id, &order, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
// The answer is heard; while it plays, its asker may still go on (within call.continueMs).
|
||||
CoreEvent::SpeakStart(id) => {
|
||||
if let Some(pid) = self.by_speech.get(id).copied() {
|
||||
for t in self.turns.values_mut().filter(|t| t.id == pid) {
|
||||
t.heard = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
CoreEvent::SpeakEnd(e) => {
|
||||
self.waiting.remove(&e.id);
|
||||
if let Some(pid) = self.by_speech.remove(&e.id) {
|
||||
self.answers.remove(&pid);
|
||||
// A held request waits for its speaker's next final even after its answer ended.
|
||||
self.turns.retain(|_, t| t.id != pid || t.held);
|
||||
for t in self.turns.values_mut().filter(|t| t.id == pid) {
|
||||
t.heard = true;
|
||||
}
|
||||
debug_log!("conversation", "answer req={pid} done played={} cancelled={} failed={}",
|
||||
e.played, e.cancelled, e.failed);
|
||||
let a = self.policy.answer_done(pid, now);
|
||||
@@ -378,6 +450,7 @@ impl<V: Voice> Conversation<V> {
|
||||
}
|
||||
CoreEvent::WorkerLost => {
|
||||
self.started.clear();
|
||||
self.turns.clear();
|
||||
for (_, (req, _)) in self.wakes.drain() {
|
||||
self.chimed.remove(&req.id);
|
||||
}
|
||||
@@ -386,9 +459,58 @@ impl<V: Voice> Conversation<V> {
|
||||
}
|
||||
}
|
||||
|
||||
/// The speaker's next utterance already started (a pause shorter than the first final's
|
||||
/// transcription): their new request is held open for it from the start.
|
||||
fn holds(&self, speaker: u64) -> bool {
|
||||
!self.calls.continue_window.is_zero() && self.started.contains_key(&speaker)
|
||||
}
|
||||
|
||||
/// A voice request goes to the policy; until its answer is heard, its speaker may go on.
|
||||
fn voice_request(&mut self, req: Request, now: Instant) {
|
||||
let held = self.holds(req.speaker);
|
||||
self.turns.insert(req.speaker, Turn { id: req.id, text: req.text.clone(), ended_at: req.ended_at, ask: None,
|
||||
held, heard: false });
|
||||
let a = self.policy.request(req, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
|
||||
/// The final `f` continues the speaker's request whose answer is not heard yet (it started within
|
||||
/// `call.continueMs` of that request's end, and is not a stop): the request is taken back, its
|
||||
/// running answer stopped. Returns its text and whether its earcon already played.
|
||||
fn going_on(&mut self, speaker: u64, f: &crate::discord::HeardFinal, now: Instant) -> Option<(String, bool)> {
|
||||
let t = self.turns.get_mut(&speaker)?;
|
||||
let started = now.checked_sub(Duration::from_millis(f.wait_ms + f.audio_ms)).unwrap_or(now);
|
||||
// An empty final (noise the worker rejected) adds nothing and asks nothing again.
|
||||
if crate::calls::normalise(&f.text).is_empty() || self.calls.continue_window.is_zero()
|
||||
|| started > t.ended_at + self.calls.continue_window
|
||||
|| matches!(self.calls.judge(speaker, &f.text), Call::Stop) {
|
||||
t.held = false;
|
||||
if t.heard {
|
||||
self.turns.remove(&speaker);
|
||||
}
|
||||
return None;
|
||||
}
|
||||
let t = self.turns.remove(&speaker).expect("checked");
|
||||
let earcon_played = self.chimed.remove(&t.id)
|
||||
|| (self.calls.earcon && (self.answers.contains_key(&t.id) || t.heard));
|
||||
debug_log!("conversation", "going on req={} speaker={speaker}", t.id);
|
||||
match t.ask {
|
||||
Some(q) => {
|
||||
self.asks.remove(&q);
|
||||
}
|
||||
None => {
|
||||
let a = self.policy.withdraw(t.id);
|
||||
self.apply(a, now);
|
||||
}
|
||||
}
|
||||
Some((t.text, earcon_played))
|
||||
}
|
||||
|
||||
/// The policy's stop, and every pending "was it said to Jarvis?" question is void.
|
||||
fn stop_everything(&mut self, now: Instant) {
|
||||
self.wakes.clear();
|
||||
self.turns.clear();
|
||||
self.waiting.clear();
|
||||
let a = self.policy.stop();
|
||||
self.apply(a, now);
|
||||
self.chimed.clear(); // nothing is waiting any more
|
||||
@@ -405,6 +527,7 @@ impl<V: Voice> Conversation<V> {
|
||||
for id in late {
|
||||
if let Some((req, _)) = self.asks.remove(&id) {
|
||||
debug_log!("conversation", "addressed req={} timed out", req.id);
|
||||
self.turns.retain(|_, t| t.id != req.id);
|
||||
let a = self.policy.utterance_closed(req.speaker, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
@@ -415,32 +538,38 @@ impl<V: Voice> Conversation<V> {
|
||||
for speaker in due {
|
||||
if let Some((req, _)) = self.wakes.remove(&speaker) {
|
||||
debug_log!("conversation", "wake word alone req={} answered alone", req.id);
|
||||
let a = self.policy.request(req, now);
|
||||
self.apply(a, now);
|
||||
self.voice_request(req, now);
|
||||
}
|
||||
}
|
||||
let a = self.policy.tick(now);
|
||||
self.apply(a, now);
|
||||
self.release_due(now);
|
||||
}
|
||||
|
||||
pub fn next_deadline(&self) -> Option<Instant> {
|
||||
let asks = self.asks.values().map(|(_, at)| *at + ADDRESSED_TIMEOUT);
|
||||
let wakes = self.wakes.values().filter_map(|(_, d)| *d);
|
||||
self.policy.next_deadline().into_iter().chain(asks).chain(wakes).min()
|
||||
// Not while the asker talks: their final decides.
|
||||
let held = self.waiting.values().filter(|(s, _)| !self.started.contains_key(s)).map(|(_, at)| *at);
|
||||
self.policy.next_deadline().into_iter().chain(asks).chain(wakes).chain(held).min()
|
||||
}
|
||||
|
||||
fn apply(&mut self, actions: Vec<Action>, now: Instant) {
|
||||
for action in actions {
|
||||
match action {
|
||||
Action::Answer(r) => match {
|
||||
// The earcon plays first, then the answer as soon as it is written.
|
||||
// The earcon plays first, then the answer once it is written and its asker has
|
||||
// been quiet for call.answerWaitMs.
|
||||
if self.calls.earcon && !self.chimed.remove(&r.id) {
|
||||
self.voice.earcon();
|
||||
}
|
||||
self.voice.answer(r.speaker, &r.text, (r.ended_at, r.arrived_at))
|
||||
self.voice.answer(r.speaker, &r.text, (r.ended_at, r.arrived_at), !self.calls.answer_wait.is_zero())
|
||||
} {
|
||||
Some(sid) => {
|
||||
debug_log!("conversation", "answer req={} speaker={} speech={sid}", r.id, r.speaker);
|
||||
if !self.calls.answer_wait.is_zero() {
|
||||
self.waiting.insert(sid.clone(), (r.speaker, r.ended_at + self.calls.answer_wait));
|
||||
}
|
||||
self.by_speech.insert(sid.clone(), r.id);
|
||||
self.answers.insert(r.id, sid);
|
||||
}
|
||||
@@ -452,11 +581,13 @@ impl<V: Voice> Conversation<V> {
|
||||
Action::StopAnswer(pid) => {
|
||||
if let Some(sid) = self.answers.remove(&pid) {
|
||||
self.by_speech.remove(&sid);
|
||||
self.waiting.remove(&sid);
|
||||
self.voice.cancel(&sid);
|
||||
}
|
||||
}
|
||||
Action::Missed(r, reason) => {
|
||||
self.chimed.remove(&r.id);
|
||||
self.turns.retain(|_, t| t.id != r.id);
|
||||
debug_log!("conversation", "missed req={} speaker={} reason={reason:?}", r.id, r.speaker);
|
||||
let mut missed = self.missed.lock().unwrap();
|
||||
if missed.len() == MISSED_KEPT {
|
||||
@@ -502,6 +633,11 @@ pub async fn run<V: Voice>(mut conv: Conversation<V>, mut events: mpsc::Unbounde
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The tests call Jarvis by one wake word of their own, whatever the default is.
|
||||
fn test_calls() -> CallSettings {
|
||||
CallSettings { wake_words: vec!["자비스".into()], answer_wait: Duration::ZERO, ..CallSettings::default() }
|
||||
}
|
||||
use crate::discord::{Heard, HeardFinal};
|
||||
use crate::playback::PlaybackEnd;
|
||||
use crate::policy::PolicyKind;
|
||||
@@ -512,6 +648,9 @@ mod tests {
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
enum Did {
|
||||
Answer(u64, String),
|
||||
/// The answer just asked for is held until released.
|
||||
Held(String),
|
||||
Release(String),
|
||||
Cancel(String),
|
||||
StopAll,
|
||||
Earcon,
|
||||
@@ -536,10 +675,17 @@ mod tests {
|
||||
people: Rc<RefCell<Vec<u64>>>,
|
||||
}
|
||||
impl Voice for Fake {
|
||||
fn answer(&self, user: u64, text: &str, _heard: (Instant, Instant)) -> Option<String> {
|
||||
fn answer(&self, user: u64, text: &str, _heard: (Instant, Instant), hold: bool) -> Option<String> {
|
||||
*self.n.borrow_mut() += 1;
|
||||
let id = format!("s{}", self.n.borrow());
|
||||
self.did.borrow_mut().push(Did::Answer(user, text.into()));
|
||||
Some(format!("s{}", self.n.borrow()))
|
||||
if hold {
|
||||
self.did.borrow_mut().push(Did::Held(id.clone()));
|
||||
}
|
||||
Some(id)
|
||||
}
|
||||
fn release(&self, id: &str) {
|
||||
self.did.borrow_mut().push(Did::Release(id.into()));
|
||||
}
|
||||
fn cancel(&self, id: &str) {
|
||||
self.did.borrow_mut().push(Did::Cancel(id.into()));
|
||||
@@ -585,10 +731,11 @@ mod tests {
|
||||
|
||||
const A: u64 = 11;
|
||||
const B: u64 = 22;
|
||||
const C: u64 = 33;
|
||||
|
||||
fn conv(kind: PolicyKind, mode: CallMode) -> (Conversation<Fake>, Fake, Instant) {
|
||||
let f = Fake::default();
|
||||
let calls = CallSettings { mode, earcon: false, ..CallSettings::default() };
|
||||
let calls = CallSettings { mode, earcon: false, ..test_calls() };
|
||||
let policy = PolicySettings { kind, collect: Duration::ZERO, ..PolicySettings::default() };
|
||||
*f.people.borrow_mut() = vec![A, B];
|
||||
(Conversation::new(f.clone(), calls, policy), f, Instant::now())
|
||||
@@ -596,7 +743,7 @@ mod tests {
|
||||
fn conv_earcon(kind: PolicyKind) -> (Conversation<Fake>, Fake, Instant) {
|
||||
let f = Fake::default();
|
||||
let policy = PolicySettings { kind, collect: Duration::ZERO, ..PolicySettings::default() };
|
||||
(Conversation::new(f.clone(), CallSettings::default(), policy), f, Instant::now())
|
||||
(Conversation::new(f.clone(), test_calls(), policy), f, Instant::now())
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -608,7 +755,7 @@ mod tests {
|
||||
assert!(f.take().is_empty(), "a waiting request gets its earcon when its answer starts");
|
||||
c.on_event(&end("s1", false), ms(t, 3000));
|
||||
assert_eq!(f.take(), vec![Did::Earcon, Did::Answer(B, "자비스 불 꺼 줘".into())]);
|
||||
c.on_event(&fin(B, "오늘 뭐 먹지", 100), ms(t, 3100));
|
||||
c.on_event(&fin(C, "오늘 뭐 먹지", 100), ms(t, 3100));
|
||||
assert!(f.take().is_empty(), "no call, no earcon");
|
||||
}
|
||||
|
||||
@@ -624,7 +771,7 @@ mod tests {
|
||||
// Nothing followed the next one: it is answered alone, still with one earcon.
|
||||
c.on_event(&end("s1", false), ms(t, 5000));
|
||||
c.on_event(&fin_utt(A, 3, "자비스"), ms(t, 6000));
|
||||
c.tick(ms(t, 6000) + CallSettings::default().follow_up);
|
||||
c.tick(ms(t, 6000) + test_calls().follow_up);
|
||||
assert_eq!(f.take(), vec![Did::Earcon, Did::Answer(A, "자비스".into())]);
|
||||
}
|
||||
|
||||
@@ -654,10 +801,10 @@ mod tests {
|
||||
c.on_event(&fin_utt(B, 1, "자비스"), ms(t, 100));
|
||||
c.on_event(&partial(B, "지금 몇 시"), ms(t, 300));
|
||||
assert_eq!(f.take(), vec![Did::Prepare(B, "자비스 지금 몇 시".into())]);
|
||||
// Not while an answer runs.
|
||||
// Not while an answer runs (to someone else: its asker going on is joined to it).
|
||||
c.on_event(&fin(A, "자비스 날씨", 100), ms(t, 400));
|
||||
f.take();
|
||||
c.on_event(&partial(A, "자비스 내일은"), ms(t, 500));
|
||||
c.on_event(&partial(C, "자비스 내일은"), ms(t, 500));
|
||||
assert!(f.take().is_empty());
|
||||
}
|
||||
|
||||
@@ -708,7 +855,7 @@ mod tests {
|
||||
c.on_event(&start(A, 1), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 600)); // the final of utterance 1 comes after 2 started
|
||||
c.on_event(&fin_utt(A, 1, "자비스"), ms(t, 900));
|
||||
c.tick(ms(t, 900) + CallSettings::default().follow_up + Duration::from_millis(1));
|
||||
c.tick(ms(t, 900) + test_calls().follow_up + Duration::from_millis(1));
|
||||
assert!(f.take().is_empty());
|
||||
c.on_event(&fin_utt(A, 2, "멈춰"), ms(t, 3000));
|
||||
assert_eq!(f.take(), vec![Did::StopAll], "joined, it is a stop");
|
||||
@@ -719,7 +866,7 @@ mod tests {
|
||||
let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord);
|
||||
c.on_event(&start(A, 1), ms(t, 0));
|
||||
c.on_event(&fin_utt(A, 1, "자비스"), ms(t, 900));
|
||||
let wait = CallSettings::default().follow_up;
|
||||
let wait = test_calls().follow_up;
|
||||
assert_eq!(c.next_deadline(), Some(ms(t, 900) + wait));
|
||||
c.tick(ms(t, 900) + wait - Duration::from_millis(1));
|
||||
assert!(f.take().is_empty());
|
||||
@@ -748,7 +895,7 @@ mod tests {
|
||||
assert!(f.take().is_empty(), "no wake word: not a call");
|
||||
c.on_event(&start(A, 2), ms(t, 62_000));
|
||||
c.on_event(&fin_utt(A, 2, "자비스"), ms(t, 63_000));
|
||||
let wait = CallSettings::default().follow_up;
|
||||
let wait = test_calls().follow_up;
|
||||
assert_eq!(c.next_deadline(), Some(ms(t, 63_000) + wait));
|
||||
c.tick(ms(t, 63_000) + wait);
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스".into())]);
|
||||
@@ -759,7 +906,7 @@ mod tests {
|
||||
let (mut c, f, t) = conv(PolicyKind::First, CallMode::WakeWord);
|
||||
c.on_event(&fin_utt(A, 1, "자비스"), ms(t, 900));
|
||||
c.on_event(&CoreEvent::Stopped, ms(t, 1000));
|
||||
c.tick(ms(t, 900) + CallSettings::default().follow_up);
|
||||
c.tick(ms(t, 900) + test_calls().follow_up);
|
||||
assert!(f.take().is_empty());
|
||||
assert_eq!(c.next_deadline(), None);
|
||||
}
|
||||
@@ -826,7 +973,7 @@ mod tests {
|
||||
let f = Fake::default();
|
||||
*f.people.borrow_mut() = vec![A, B, 33];
|
||||
let calls = CallSettings { mode: CallMode::Addressed, earcon: false, designated: [A].into(), designated_only: true,
|
||||
..CallSettings::default() };
|
||||
..test_calls() };
|
||||
let mut c = Conversation::new(f.clone(), calls, PolicySettings { kind: PolicyKind::InOrder, ..PolicySettings::default() });
|
||||
c.on_event(&fin(A, "불 좀 꺼 줄래", 100), ms(t, 0));
|
||||
assert!(matches!(f.take()[..], [Did::Addressed(_, A)]));
|
||||
@@ -899,7 +1046,7 @@ mod tests {
|
||||
let (mut c, f, t) = conv(PolicyKind::InOrder, CallMode::Addressed);
|
||||
c.on_event(&fin(A, "자비스 날씨", 100), ms(t, 0));
|
||||
c.on_event(&fin(B, "자비스 뉴스", 100), ms(t, 100));
|
||||
c.on_event(&fin(B, "불 좀 꺼 줄래", 100), ms(t, 200)); // waiting for "addressed"
|
||||
c.on_event(&fin(C, "불 좀 꺼 줄래", 100), ms(t, 200)); // waiting for "addressed"
|
||||
let did = f.take();
|
||||
let q = did.iter().find_map(|d| if let Did::Addressed(q, _) = d { Some(q.clone()) } else { None }).unwrap();
|
||||
// The slash command stops all speech: Stopped comes before the speech's end.
|
||||
@@ -930,7 +1077,7 @@ mod tests {
|
||||
(CH_ALL, ChatChannelCall { everything: true, send: true }),
|
||||
(CH_MUTE, ChatChannelCall { everything: true, send: false }),
|
||||
].into() };
|
||||
let voice = CallSettings { earcon: false, ..CallSettings::default() };
|
||||
let voice = CallSettings { earcon: false, ..test_calls() };
|
||||
let c = Conversation::new(f.clone(), voice, PolicySettings::default()).with_chat(chat);
|
||||
(c, f, Instant::now())
|
||||
}
|
||||
@@ -942,7 +1089,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn a_chat_call_is_answered_as_a_reply_to_its_message() {
|
||||
let (mut c, f, t) = chat_conv(CallSettings::default());
|
||||
let (mut c, f, t) = chat_conv(test_calls());
|
||||
c.on_event(&chat(CH_WAKE, 5, A, false, "자비스 내일 일정 알려 줘"), t);
|
||||
let Did::ChatAnswer(id, user, text) = f.take().remove(0) else { panic!() };
|
||||
assert_eq!((user, text.as_str()), (A, "자비스 내일 일정 알려 줘"));
|
||||
@@ -956,7 +1103,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn chat_answers_follow_each_channels_switch_and_never_answer_bots() {
|
||||
let (mut c, f, t) = chat_conv(CallSettings::default());
|
||||
let (mut c, f, t) = chat_conv(test_calls());
|
||||
c.on_event(&chat(CH_WAKE, 1, A, false, "오늘 뭐 먹지"), t);
|
||||
assert!(f.take().is_empty(), "wake word only");
|
||||
c.on_event(&chat(CH_ALL, 2, A, false, "오늘 뭐 먹지"), t);
|
||||
@@ -974,7 +1121,7 @@ mod tests {
|
||||
#[test]
|
||||
fn chat_uses_its_own_wake_words_and_the_designated_people() {
|
||||
let calls = CallSettings { wake_words: vec!["봇".into()], designated: [A].into(), designated_only: true,
|
||||
..CallSettings::default() };
|
||||
..test_calls() };
|
||||
let (mut c, f, t) = chat_conv(calls);
|
||||
c.on_event(&chat(CH_WAKE, 1, A, false, "봇 안녕"), t);
|
||||
assert_eq!(f.take().len(), 1);
|
||||
@@ -986,7 +1133,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn a_failed_chat_answer_posts_nothing_and_chat_leaves_voice_alone() {
|
||||
let (mut c, f, t) = chat_conv(CallSettings::default());
|
||||
let (mut c, f, t) = chat_conv(test_calls());
|
||||
c.on_event(&fin(A, "자비스 날씨", 100), t);
|
||||
c.on_event(&chat(CH_ALL, 9, B, false, "안녕"), t);
|
||||
let did = f.take();
|
||||
@@ -999,7 +1146,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn a_chat_answer_after_its_keep_time_is_not_posted() {
|
||||
let (mut c, f, t) = chat_conv(CallSettings::default());
|
||||
let (mut c, f, t) = chat_conv(test_calls());
|
||||
c.on_event(&chat(CH_WAKE, 5, A, false, "자비스 안녕"), t);
|
||||
let Did::ChatAnswer(id, ..) = f.take().remove(0) else { panic!() };
|
||||
c.on_event(&CoreEvent::ReplyText { id: id.clone(), text: "안녕하세요.".into(), failed: false }, t + CHAT_ANSWER_KEPT);
|
||||
@@ -1011,9 +1158,261 @@ mod tests {
|
||||
let (mut c, f, t) = conv(PolicyKind::InOrder, CallMode::WakeWord);
|
||||
c.on_event(&fin(A, "자비스 하나", 0), ms(t, 0));
|
||||
c.on_event(&fin(B, "자비스 둘", 0), ms(t, 1000));
|
||||
c.on_event(&fin(A, "자비스 셋", 900), ms(t, 1500)); // ended at 600, before 둘
|
||||
c.on_event(&fin(C, "자비스 셋", 900), ms(t, 1500)); // ended at 600, before 둘
|
||||
f.take();
|
||||
c.on_event(&end("s1", false), ms(t, 2000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스 셋".into())]);
|
||||
assert_eq!(f.take(), vec![Did::Answer(C, "자비스 셋".into())]);
|
||||
}
|
||||
|
||||
fn alone(kind: PolicyKind) -> (Conversation<Fake>, Fake, Instant) {
|
||||
let (c, f, t) = conv(kind, CallMode::Addressed);
|
||||
*f.people.borrow_mut() = vec![A];
|
||||
(c, f, t)
|
||||
}
|
||||
fn speak_start(id: &str) -> CoreEvent {
|
||||
CoreEvent::SpeakStart(id.into())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_speaker_who_goes_on_before_the_answer_is_heard_gets_one_answer_for_all_of_it() {
|
||||
// Live 2026-11-19: each piece of one thought was answered on its own, one after another.
|
||||
for kind in [PolicyKind::Latest, PolicyKind::InOrder, PolicyKind::First, PolicyKind::Important] {
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into())], "{kind:?}");
|
||||
// The next piece starts 0.8 s after the first ended, while Jarvis is still thinking.
|
||||
c.on_event(&fin(A, "디코 화면 공유 창에 가려져 있잖아", 100), ms(t, 1800));
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())], "{kind:?}");
|
||||
c.on_event(&fin(A, "앞으로 오게 해 줘", 100), ms(t, 4000));
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s2".into()),
|
||||
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아 앞으로 오게 해 줘".into())],
|
||||
"{kind:?}");
|
||||
assert_eq!(c.missed().count(), 0, "a joined piece is not a missed request ({kind:?})");
|
||||
// The cancelled speech's end does not end the joined answer.
|
||||
c.on_event(&end("s1", true), ms(t, 4100));
|
||||
assert!(f.take().is_empty(), "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_follows_a_wake_word_call_is_joined_even_without_the_wake_word() {
|
||||
let (mut c, f, t) = conv(PolicyKind::Latest, CallMode::WakeWord);
|
||||
c.on_event(&fin(A, "자비스 디코 화면 공유 보여", 100), ms(t, 0));
|
||||
c.on_event(&fin(A, "내 화면 말이야", 100), ms(t, 2000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스 디코 화면 공유 보여".into()), Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "자비스 디코 화면 공유 보여 내 화면 말이야".into())]);
|
||||
// Someone else's words never join it.
|
||||
c.on_event(&fin(B, "나도 안 보여", 100), ms(t, 2500));
|
||||
assert!(f.take().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn once_the_answer_ended_or_the_pause_is_long_the_next_words_are_a_new_request() {
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1500));
|
||||
c.on_event(&end("s1", false), ms(t, 2000));
|
||||
c.on_event(&fin(A, "고마워", 100), ms(t, 3500));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into()), Did::Answer(A, "고마워".into())]);
|
||||
// Still thinking, but the speaker paused longer than call.continueMs.
|
||||
let (mut c, f, t) = alone(PolicyKind::InOrder);
|
||||
let pause = test_calls().continue_window.as_millis() as u64;
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
// The next utterance (1 s long) starts `pause` + 100 ms after the first ended.
|
||||
c.on_event(&fin(A, "메모장도", 100), ms(t, pause + 1100));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into())], "queued as its own request");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stop_said_while_the_answer_is_prepared_still_stops_it() {
|
||||
let (mut c, f, t) = conv(PolicyKind::Latest, CallMode::WakeWord);
|
||||
c.on_event(&fin(A, "자비스 날씨 알려 줘", 100), ms(t, 0));
|
||||
c.on_event(&fin(A, "자비스 멈춰", 100), ms(t, 1500));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스 날씨 알려 줘".into()), Did::StopAll, Did::Cancel("s1".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_that_started_before_the_answer_played_is_still_joined() {
|
||||
// The next utterance starts while Jarvis thinks; the answer starts playing before its final.
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1500));
|
||||
c.on_event(&fin_utt(A, 2, "디코 화면 공유 창에 가려져 있잖아"), ms(t, 3000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into()), Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_final_leaves_the_request_as_it_is() {
|
||||
// Noise the worker rejected comes as an empty final: nothing to join, nothing asked again.
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
c.on_event(&fin(A, "", 100), ms(t, 1500));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into())]);
|
||||
// An empty final after the answer played ends the hold: what follows is a new request.
|
||||
c.on_event(&start(A, 2), ms(t, 2000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 2500));
|
||||
c.on_event(&fin_utt(A, 2, " "), ms(t, 3200));
|
||||
c.on_event(&fin(A, "고마워", 100), ms(t, 4500));
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s1".into()), Did::Answer(A, "고마워".into())]);
|
||||
}
|
||||
|
||||
const ALL: [PolicyKind; 4] = [PolicyKind::Latest, PolicyKind::InOrder, PolicyKind::First, PolicyKind::Important];
|
||||
|
||||
/// Alone with Jarvis, answers held until the asker has been quiet for `wait_ms`.
|
||||
fn alone_waiting(kind: PolicyKind, wait_ms: u64) -> (Conversation<Fake>, Fake, Instant) {
|
||||
let f = Fake::default();
|
||||
let calls = CallSettings { mode: CallMode::Addressed, earcon: false, answer_wait: Duration::from_millis(wait_ms),
|
||||
..test_calls() };
|
||||
let policy = PolicySettings { kind, collect: Duration::ZERO, ..PolicySettings::default() };
|
||||
*f.people.borrow_mut() = vec![A];
|
||||
(Conversation::new(f.clone(), calls, policy), f, Instant::now())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_answer_starts_only_after_its_asker_was_quiet_for_the_wait() {
|
||||
// Live 2026-11-19: pauses inside one thought were up to 4.7 s, longer than writing an answer.
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0)); // ended at -100
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into()), Did::Held("s1".into())], "{kind:?}");
|
||||
assert_eq!(c.next_deadline(), Some(ms(t, 2900)), "{kind:?}");
|
||||
c.tick(ms(t, 2800));
|
||||
assert!(f.take().is_empty(), "{kind:?}");
|
||||
c.tick(ms(t, 2900));
|
||||
assert_eq!(f.take(), vec![Did::Release("s1".into())], "{kind:?}");
|
||||
assert_eq!(c.next_deadline(), None, "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_answer_waits_while_its_asker_talks_and_their_words_join_it() {
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
f.take();
|
||||
c.on_event(&start(A, 2), ms(t, 2000));
|
||||
c.tick(ms(t, 2900));
|
||||
assert!(f.take().is_empty(), "still talking ({kind:?})");
|
||||
c.on_event(&fin_utt(A, 2, "메모장도"), ms(t, 3500));
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s1".into()), Did::Answer(A, "계산기 켜 줘 메모장도".into()),
|
||||
Did::Held("s2".into())], "{kind:?}");
|
||||
c.tick(ms(t, 6400)); // the joined request ended at 3400
|
||||
assert_eq!(f.take(), vec![Did::Release("s2".into())], "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_answer_is_released_when_what_followed_was_noise() {
|
||||
let (mut c, f, t) = alone_waiting(PolicyKind::Latest, 3000);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 2000));
|
||||
c.tick(ms(t, 2900));
|
||||
f.take();
|
||||
c.on_event(&fin_utt(A, 2, " "), ms(t, 3500));
|
||||
assert_eq!(f.take(), vec![Did::Release("s1".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_while_the_answer_plays_stops_it_and_answers_all_of_it() {
|
||||
// A pause longer than the wait: the answer started, then the speaker went on within
|
||||
// call.continueMs of their request's end.
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone_waiting(kind, 3000);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
c.tick(ms(t, 2900));
|
||||
c.on_event(&speak_start("s1"), ms(t, 3000));
|
||||
f.take();
|
||||
c.on_event(&fin(A, "제일 앞으로", 100), ms(t, 5000)); // started at 3900, 4 s after the end
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s1".into()), Did::Answer(A, "계산기 켜 줘 제일 앞으로".into()),
|
||||
Did::Held("s2".into())], "{kind:?}");
|
||||
assert_eq!(c.missed().count(), 0, "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_is_joined_even_when_the_short_answer_already_ended() {
|
||||
// Final -> next SpeechStart -> the answer plays and ends -> next Final.
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1500));
|
||||
c.on_event(&end("s1", false), ms(t, 2500));
|
||||
c.on_event(&fin_utt(A, 2, "디코 화면 공유 창에 가려져 있잖아"), ms(t, 4000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into()),
|
||||
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())], "{kind:?}");
|
||||
assert_eq!(c.missed().count(), 0, "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_that_started_before_the_first_final_is_joined() {
|
||||
// A very short pause: the next utterance starts before the first one's final arrives.
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&start(A, 1), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&fin_utt(A, 1, "지금 화면 봐봐"), ms(t, 1100));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1600));
|
||||
c.on_event(&fin_utt(A, 2, "디코 화면 공유 창에 가려져 있잖아"), ms(t, 3000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into()), Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())], "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_request_whose_next_final_is_empty_or_a_stop_is_not_asked_again() {
|
||||
for kind in ALL {
|
||||
for last in ["", "자비스 멈춰"] {
|
||||
// (a) the answer already ended.
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1500));
|
||||
c.on_event(&end("s1", false), ms(t, 2500));
|
||||
c.on_event(&fin_utt(A, 2, last), ms(t, 4000));
|
||||
let did = f.take();
|
||||
assert_eq!(did.iter().filter(|d| matches!(d, Did::Answer(..))).count(), 1, "{kind:?} {last:?}");
|
||||
// What follows is a new request, not joined to the old one.
|
||||
c.on_event(&fin(A, "고마워", 100), ms(t, 5000));
|
||||
assert_eq!(f.take().into_iter().filter(|d| matches!(d, Did::Answer(..))).collect::<Vec<_>>(),
|
||||
vec![Did::Answer(A, "고마워".into())], "{kind:?} {last:?}");
|
||||
// (b) the next utterance started before the first final, the answer is playing.
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&start(A, 1), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&fin_utt(A, 1, "지금 화면 봐봐"), ms(t, 1100));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1600));
|
||||
c.on_event(&fin_utt(A, 2, last), ms(t, 3000));
|
||||
let did = f.take();
|
||||
assert_eq!(did.iter().filter(|d| matches!(d, Did::Answer(..))).count(), 1, "{kind:?} {last:?}");
|
||||
if last.is_empty() {
|
||||
assert!(!did.contains(&Did::Cancel("s1".into())), "noise does not stop the answer ({kind:?})");
|
||||
} else {
|
||||
assert!(did.contains(&Did::StopAll), "{kind:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_lets_the_brain_get_ready_for_the_joined_request() {
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
f.take();
|
||||
c.on_event(&partial(A, "제일 위로"), ms(t, 1200));
|
||||
assert_eq!(f.take(), vec![Did::Prepare(A, "계산기 켜 줘 제일 위로".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_joined_request_gets_no_second_earcon() {
|
||||
let (mut c, f, t) = conv_earcon(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "자비스 날씨", 100), ms(t, 0));
|
||||
c.on_event(&fin(A, "부산", 100), ms(t, 1500));
|
||||
assert_eq!(f.take(), vec![Did::Earcon, Did::Answer(A, "자비스 날씨".into()), Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "자비스 날씨 부산".into())]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ Settings (`settings.json`, section `call`; wake words stay in `voice.wakeWords`)
|
||||
|
||||
| Key | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `voice.wakeWords` | `["자비스"]` | Wake words for voice (also the STT hint). |
|
||||
| `voice.wakeWords` | `["비서", "비서야", "비서님", "김비서", "김비서야", "김비서님"]` | Wake words for voice (also the STT hint). The default calls Jarvis like a secretary, by the word and its usual call forms (user request, 2026-11-19; before it was `["자비스"]`). |
|
||||
| `call.chatWakeWords` | `null` | Wake words for chat; `null` uses `voice.wakeWords`. |
|
||||
| `call.similarity` | `0.2` | How far a heard word may be from a wake word, as a share of the wake word's normalised letters (0-0.5). `0` needs an exact match. |
|
||||
| `call.mode` | `"wakeWord"` | `wakeWord`: answer only when called by a wake word. `addressed`: also answer what is judged to be said to Jarvis. `everything`: answer every utterance. |
|
||||
@@ -34,6 +34,8 @@ Settings (`settings.json`, section `call`; wake words stay in `voice.wakeWords`)
|
||||
| `call.stopWords` | `["멈춰", "그만", "정지", "stop"]` | Said together with a wake word, they stop Jarvis in any policy. |
|
||||
| `call.earcon` | `true` | Play the earcon (below) when Jarvis takes a voice call. |
|
||||
| `call.followUpMs` | `1500` | After a wake word said alone, how long the speaker's next utterance may take to start and still be joined to it. 0-5000. |
|
||||
| `call.continueMs` | `5000` | Going on (below): how long after a request's utterance ended the speaker's next utterance may start and still be part of it, while its answer has not ended. 0 turns joining off. 0-15000. |
|
||||
| `call.answerWaitMs` | `3000` | Waiting before answering (below): a voice answer starts playing only once its asker has been quiet this long after their request ended. 0: as soon as it is written. 0-10000. |
|
||||
|
||||
Normalisation: Unicode NFKD, lower case, and only letters and digits kept (spaces, punctuation and
|
||||
combining marks dropped). NFKD splits Hangul syllables into their letters, so "자비즈" is one letter
|
||||
@@ -42,6 +44,9 @@ away from "자비스" (of six) while "서비스" is two away.
|
||||
A wake word is heard when some stretch of the normalised text is at most
|
||||
`floor(similarity × letters of the wake word)` edits (insert, delete, substitute) from it. With the
|
||||
default 0.2, "자비스" (six letters) allows one edit. A wake word anywhere in the utterance counts.
|
||||
The earliest heard wake word is the call; where several are heard starting there, the one with the
|
||||
fewest edits, then the longest, so a call form ("비서야") is removed whole and leaves nothing over
|
||||
("비서야 멈춰" is a stop), and a near match of a longer word does not eat into what follows.
|
||||
|
||||
Decision for one utterance (`judge(speaker, text)`), in order:
|
||||
|
||||
@@ -75,6 +80,43 @@ connection (`WorkerLost`): the final it waited for will not come, and the restar
|
||||
utterances are not continuations of it. Chat messages are not joined:
|
||||
a message is complete as sent.
|
||||
|
||||
**Going on.** People say one thing in pieces, with pauses of a fraction of a second to several
|
||||
seconds between them (live 2026-11-19: 0.1-4.7 s inside one thought), and every pause longer than
|
||||
the worker's `end_silence_ms` ends an utterance. Each piece used to be answered on its own, one after
|
||||
another, and the answers piled up (user request, 2026-11-19: what is said in pieces is one request).
|
||||
So a speaker's voice request stays open until its answer has ended (`SpeakEnd`; also while it
|
||||
plays, since live 2026-11-19 pauses of 2.3-4.7 s were longer than writing the answer and the answer
|
||||
started in the pause), and while it is open their next final is joined to it when that utterance
|
||||
started within `call.continueMs` of the request's end (start = the final's arrival minus its
|
||||
`wait_ms` and `audio_ms`):
|
||||
- The earlier request is taken back (the policy's `withdraw`): a running answer is stopped (its
|
||||
speech `cancel`led, which also stops the brain, `voice_worker.spec.md`), a waiting one leaves the
|
||||
queue, a pending "addressed" question is void. It is not a missed request.
|
||||
- The joined text (the earlier text, a space, the new final) is judged and answered as one request,
|
||||
as if it had been said in one go; it ends when the new final ended. It may be joined again.
|
||||
- The new final is joined whatever it is on its own (a piece needs no wake word of its own), except
|
||||
a stop (a stop stops, as always) and an empty final (noise the worker rejected: nothing is joined
|
||||
or asked again, the request goes on as it was).
|
||||
- When the speaker's next utterance starts (`SpeechStart`) within `call.continueMs` of the request's
|
||||
end, or had already started when the request's final came (a pause shorter than the
|
||||
transcription), the request is held open for that utterance's final, also if its answer starts
|
||||
playing or even ends meanwhile; the joined request then stops that answer if it still plays. An
|
||||
empty final or a stop ends the hold, and nothing is asked again.
|
||||
- Other speakers' utterances never join it. An answer that ended, a missed
|
||||
request, a stop, leaving and a lost worker end it; then what the speaker says is a new request.
|
||||
- The joined request gets no second earcon when the first one's already played.
|
||||
- While a request is open, its speaker's partials are sent to get ready as the joined text, also while
|
||||
its answer is being written (it is the request that will be answered).
|
||||
|
||||
**Waiting before answering.** So that a pause inside one thought does not get an answer, a voice
|
||||
answer is written at once but starts playing only when its asker has been quiet for
|
||||
`call.answerWaitMs` after the request ended (default 3 s: 8 of the 10 pauses inside one thought
|
||||
measured live 2026-11-19 were shorter). While the asker is talking when that time comes, the answer
|
||||
waits for that utterance's final: it joins the request (Going on, the held answer is cancelled and
|
||||
the joined one waits again from its own end), or, when it is empty or not joined, the answer plays.
|
||||
A stop stops it as always. The earcon is not held: it still tells the asker at once that Jarvis
|
||||
heard them. Answers queued behind a held one wait their turn.
|
||||
|
||||
**Earcon.** A short two-tone "띠링" (about 0.3 s, made by the app, `app/src/earcon.rs`) tells the
|
||||
speaker Jarvis heard the call, long before the answer is written: it is the first sound after a call.
|
||||
It plays right before each voice answer starts (the policy's `Answer`), and at once when a wake word
|
||||
@@ -113,6 +155,9 @@ Rules shared by every policy:
|
||||
than the request's `ended_at` (in `important`: any ending utterance, since it competes on score), the
|
||||
policy waits for that final, at most `settleMs` from the first waiting request's arrival. A final that
|
||||
is not a call ends the wait for it. `latest` does not settle: an earlier utterance would lose anyway.
|
||||
- **Taking back.** A request whose speaker went on (Going on) is withdrawn: stopped when it is the
|
||||
running answer, removed when it waits, dropped from a pending ranking. Not missed; the joined
|
||||
request that follows goes through the policy as any new request.
|
||||
- **Stop.** A stop decision stops the current answer and drops every waiting request, in every policy.
|
||||
The stopped answer and the dropped requests are missed (`stopped`).
|
||||
- An answer that is already done is never started again.
|
||||
@@ -178,7 +223,8 @@ event is passed on unchanged). It talks to the core through a small `Voice` inte
|
||||
one request, stop all, earcon, prepare, ask "addressed", ask "rank"), so the glue is tested with a fake.
|
||||
|
||||
- `Chat` and `ReplyText`: chat calls, above.
|
||||
- `Partial` (getting ready): a partial transcript that call detection already reads as a request
|
||||
- `Partial` (getting ready): a partial of a speaker whose request is open (Going on) is sent as the
|
||||
joined text. Otherwise a partial transcript that call detection already reads as a request
|
||||
(joined to the speaker's waiting wake word alone, as its final would be; not the wake word alone,
|
||||
not a stop) is sent to the brain as `prepare(user, text)`, so the brain judges what the call needs
|
||||
while the speaker is still talking (`voice_worker.spec.md`, `prepare`). Not while an answer runs:
|
||||
@@ -191,10 +237,13 @@ one request, stop all, earcon, prepare, ask "addressed", ask "rank"), so the glu
|
||||
judgement) closes the utterance, and no answer within 5 s counts as no, also when the answer arrives
|
||||
after that but before the timer ran.
|
||||
- Policy actions: `Answer` → the earcon (unless it already played for this request), then
|
||||
`answer(user, text)` (the brain answers and the answer is spoken; when it
|
||||
cannot be queued the answer is done at once); `StopAnswer` → `cancel` of that answer's speech;
|
||||
`answer(user, text, hold)` (the brain answers and the answer is spoken; held, with a non-zero
|
||||
`call.answerWaitMs`, until `release` when its asker has been quiet for it; when it cannot be
|
||||
queued the answer is done at once); `StopAnswer` → `cancel` of that answer's speech;
|
||||
`Rank { id }` → `rank` with id `k<id>` and the candidates' request ids; `Missed` → a debug log and
|
||||
the missed list (the last 100: speaker, reason, time and length, never the text).
|
||||
- `SpeakStart` of an answer's speech: the answer is heard (no second earcon when it is joined); its
|
||||
request stays open until the answer ends (Going on).
|
||||
- `SpeakEnd` of an answer's speech is that answer's done. The end of a speech the loop already
|
||||
cancelled starts nothing.
|
||||
- `Stopped` (all speech stopped from elsewhere: the slash command, `stop()`, the session ending) and
|
||||
@@ -208,14 +257,21 @@ each to the chat model, the heard text fenced as data. Eval: `evals/test_voice_j
|
||||
## Tests
|
||||
|
||||
`cargo test` covers call detection (exact and near wake words in Hangul and Latin, a different word
|
||||
two letters away, wake word anywhere, stop with and without a wake word, designated combinations,
|
||||
two letters away, wake word anywhere, the longest call form where several start (the default secretary call forms), stop with and without a wake word, designated combinations,
|
||||
modes) and every policy rule above with fake times (settling, ordering by end time, busy, queue limit,
|
||||
interruption and `response.interrupt` off, ranking tie, ranking timeout, stop dropping the queue), and the conversation loop with a
|
||||
fake voice side (answering and queueing, a busy drop recorded without text, voice stop, addressed alone
|
||||
with Jarvis answered without asking, the room counts sent with the question, addressed yes,
|
||||
no and timeout, the earcon before each answer, once for a wake word alone, none while busy or when
|
||||
off, partials that read as a call sent to get ready (joined to a waiting wake word, none while
|
||||
answering), interruption cancelling the right speech, a ranking deciding, end times from
|
||||
answering someone else, the joined text while going on), going on (pieces joined into one answer in
|
||||
every policy without a missed record, a piece without a wake word joined, other speakers never, a
|
||||
new request once the answer ended or the pause is longer than `call.continueMs`, joined while the
|
||||
answer plays, answers held until the asker was quiet for `call.answerWaitMs`, longer while they talk
|
||||
and joined by what they say, released after noise, in every policy, a stop still
|
||||
stopping, no second earcon, an utterance started before the answer played or before the first final
|
||||
came still joined, also after a short answer ended, in every policy; an empty final or a stop
|
||||
after a hold asking nothing again), interruption cancelling the right speech, a ranking deciding, end times from
|
||||
`wait_ms`, and chat: an answer posted as a reply once, per-channel switches, no answers to bots or to
|
||||
a stop, chat wake words and designated people, a failed answer posting nothing). `tests/worker_protocol.rs` checks an answer spoken under its id and both judgements against
|
||||
the real Python worker with a fake brain.
|
||||
|
||||
@@ -207,6 +207,8 @@ struct Inner {
|
||||
notes: OnceLock<Arc<crate::notes::Notes>>,
|
||||
/// What Jarvis did outside its answers (admin.spec.md, "Activity").
|
||||
activity: OnceLock<Arc<crate::activity::Activity>>,
|
||||
/// PC control, when it is available (admin.spec.md, "PC control").
|
||||
pc: OnceLock<Arc<crate::pc::Pc>>,
|
||||
/// Speech requests whose end someone waits for (`say_to_end`), by request id.
|
||||
speech_waits: Mutex<HashMap<String, tokio::sync::oneshot::Sender<PlaybackEnd>>>,
|
||||
/// The spoken transcription notice of the current voice session.
|
||||
@@ -334,6 +336,7 @@ impl DiscordCore {
|
||||
notice: Mutex::new(JoinNotice::default()),
|
||||
activity: OnceLock::new(),
|
||||
notes: OnceLock::new(),
|
||||
pc: OnceLock::new(),
|
||||
});
|
||||
tokio::spawn(player_task(Arc::downgrade(&inner), player_rx));
|
||||
tokio::spawn(worker_task(Arc::downgrade(&inner), worker_events));
|
||||
@@ -560,15 +563,26 @@ impl DiscordCore {
|
||||
/// `SpeakStart` and `SpeakEnd` follow as for `say`), or None when not in a voice channel. It ends as
|
||||
/// failed when the brain gives no answer. `heard`: when the request's speech ended and its final
|
||||
/// arrived, passed on in `AnswerAsked`.
|
||||
pub fn answer(&self, user: u64, text: &str, heard: (Instant, Instant)) -> Option<String> {
|
||||
pub fn answer(&self, user: u64, text: &str, heard: (Instant, Instant), hold: bool) -> Option<String> {
|
||||
self.voice_channel_id()?;
|
||||
let id = with_playback(&self.inner, |p| p.adopt());
|
||||
let id = with_playback(&self.inner, |p| {
|
||||
let id = p.adopt();
|
||||
if hold {
|
||||
p.hold(&id);
|
||||
}
|
||||
id
|
||||
});
|
||||
let _ = self.inner.events.send(CoreEvent::AnswerAsked { id: id.clone(), user, ended_at: heard.0,
|
||||
final_at: heard.1 });
|
||||
self.inner.worker.reply_in(&id, user, text, "voice", true, None, ¬es_of(&self.inner, user));
|
||||
Some(id)
|
||||
}
|
||||
|
||||
/// Let an answer held by `answer(.., hold)` play.
|
||||
pub fn release(&self, id: &str) {
|
||||
with_playback(&self.inner, |p| p.release(id));
|
||||
}
|
||||
|
||||
/// Play the earcon in the voice channel, queued like speech. The request id, or None when not in a
|
||||
/// voice channel.
|
||||
pub fn earcon(&self) -> Option<String> {
|
||||
@@ -597,6 +611,11 @@ impl DiscordCore {
|
||||
let _ = self.inner.activity.set(activity);
|
||||
}
|
||||
|
||||
/// PC control for the brain's pcControl tool; without it its actions are refused.
|
||||
pub fn set_pc(&self, pc: Arc<crate::pc::Pc>) {
|
||||
let _ = self.inner.pc.set(pc);
|
||||
}
|
||||
|
||||
/// Run a scheduled task in the brain (read-only tools, not spoken) and wait for its answer.
|
||||
pub async fn run_task(&self, id: &str, user: u64, text: &str) -> Option<String> {
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
@@ -848,6 +867,68 @@ fn report_voice_state(inner: &Inner) {
|
||||
debug_log!("voice", "reported voice {channel} with {n} members to the worker");
|
||||
}
|
||||
|
||||
/// One `screen_action` against the real screen.
|
||||
async fn screen_act(inner: &Arc<Inner>, req: crate::worker_client::ScreenAction) -> Result<serde_json::Value, String> {
|
||||
let settings = inner.settings.clone();
|
||||
tokio::task::spawn_blocking(move || match crate::screen::system_screen() {
|
||||
Some(screen) => crate::screen::act(&settings, screen.as_ref(), &req),
|
||||
None => Err("이 PC에서는 화면을 볼 수 없습니다".into()),
|
||||
}).await.unwrap_or_else(|_| Err("화면을 캡처하지 못했습니다".into()))
|
||||
}
|
||||
|
||||
/// How long a voice request waits for a screen share to be popped out, and how often it looks.
|
||||
const SHARE_WAIT: Duration = Duration::from_secs(30);
|
||||
const SHARE_POLL: Duration = Duration::from_secs(1);
|
||||
|
||||
/// Jarvis's voice channel's name and the people sharing their screen there (ID, display name);
|
||||
/// None outside a voice channel.
|
||||
fn share_state(inner: &Inner) -> Option<(Option<String>, Vec<(u64, String)>)> {
|
||||
let (guild, channel) = inner.session.lock().unwrap().as_ref().map(|s| (s.guild_id, s.channel_id))?;
|
||||
// IDs first, then names: the cache's guild is not held while `name_of` reads it again.
|
||||
let (name, users): (Option<String>, Vec<u64>) = inner.cache.get()
|
||||
.and_then(|c| c.guild(GuildId::new(guild)).map(|g| (
|
||||
g.channels.get(&ChannelId::new(channel)).map(|c| c.name.clone()),
|
||||
g.voice_states.values()
|
||||
.filter(|v| v.channel_id.map(|c| c.get()) == Some(channel) && v.self_stream == Some(true))
|
||||
.map(|v| v.user_id.get())
|
||||
.collect())))
|
||||
.unwrap_or_default();
|
||||
let bot = inner.bot_id.get().copied();
|
||||
Some((name, users.into_iter().filter(|u| Some(*u) != bot).map(|u| (u, name_of(inner, u))).collect()))
|
||||
}
|
||||
|
||||
/// Someone's Discord screen share (admin.spec.md, "Screen viewing"): only while the person asked
|
||||
/// about shares their screen in Jarvis's voice channel, from the window Discord pops it out to. With
|
||||
/// no such window a voice request asks for it in the channel and waits `SHARE_WAIT`.
|
||||
async fn share_capture(inner: &Arc<Inner>, mut req: crate::worker_client::ScreenAction) -> Result<serde_json::Value, String> {
|
||||
// Who may see comes first: nothing is said or waited for for someone who may not.
|
||||
if let Some(why) = crate::screen::refusal(&inner.settings, &req.user) {
|
||||
return Err(why);
|
||||
}
|
||||
let (channel, streaming) = share_state(inner).ok_or("자비스가 음성 채널에 있을 때만 화면공유를 볼 수 있습니다")?;
|
||||
let name = crate::screen::share_streamer(&streaming, req.person.as_deref())?;
|
||||
req.voice_channel = channel;
|
||||
let first = screen_act(inner, req.clone()).await;
|
||||
if first.as_ref().err().map(String::as_str) != Some(crate::screen::NO_SHARE_WINDOW) {
|
||||
return first;
|
||||
}
|
||||
let prompt = crate::screen::share_prompt(name.as_deref());
|
||||
if req.via.as_deref() != Some("voice") {
|
||||
return Err(format!("{} 그 뒤 다시 물어봐 주세요", prompt.trim_end_matches('.')));
|
||||
}
|
||||
debug_log!("conversation", "screen share {}: no popped-out window, asking and waiting", req.id);
|
||||
with_playback(inner, |p| p.say(&prompt, false));
|
||||
let until = tokio::time::Instant::now() + SHARE_WAIT;
|
||||
while tokio::time::Instant::now() < until {
|
||||
tokio::time::sleep(SHARE_POLL).await;
|
||||
let r = screen_act(inner, req.clone()).await;
|
||||
if r.as_ref().err().map(String::as_str) != Some(crate::screen::NO_SHARE_WINDOW) {
|
||||
return r;
|
||||
}
|
||||
}
|
||||
Err("화면공유 창이 분리되지 않아 볼 수 없습니다".into())
|
||||
}
|
||||
|
||||
/// The registered chat channels and their switches, for the brain's chat tool.
|
||||
fn report_chat_channels(inner: &Inner) {
|
||||
inner.worker.chat_channels(inner.bot_id.get().copied(), &inner.settings.chat_channels);
|
||||
@@ -995,12 +1076,11 @@ async fn worker_task(core: std::sync::Weak<Inner>, mut rx: mpsc::UnboundedReceiv
|
||||
let inner = inner.clone();
|
||||
tokio::spawn(async move {
|
||||
let started = std::time::Instant::now();
|
||||
let settings = inner.settings.clone();
|
||||
let req = a.clone();
|
||||
let result = tokio::task::spawn_blocking(move || match crate::screen::system_screen() {
|
||||
Some(screen) => crate::screen::act(&settings, screen.as_ref(), &req),
|
||||
None => Err("이 PC에서는 화면을 볼 수 없습니다".into()),
|
||||
}).await.unwrap_or_else(|_| Err("화면을 캡처하지 못했습니다".into()));
|
||||
let result = if a.action == "capture" && a.target.as_deref() == Some("share") {
|
||||
share_capture(&inner, a.clone()).await
|
||||
} else {
|
||||
screen_act(&inner, a.clone()).await
|
||||
};
|
||||
// The kind of target, size and time only: never a title or a picture.
|
||||
let target = crate::screen::target_of(&a).map(|(t, _)| crate::screen::activity_target(&t))
|
||||
.unwrap_or_else(|_| "?".into());
|
||||
@@ -1018,6 +1098,26 @@ async fn worker_task(core: std::sync::Weak<Inner>, mut rx: mpsc::UnboundedReceiv
|
||||
inner.worker.screen_action_result(&a.id, result);
|
||||
});
|
||||
}
|
||||
WorkerEvent::PcAction(a) => {
|
||||
let inner = inner.clone();
|
||||
tokio::spawn(async move {
|
||||
let started = std::time::Instant::now();
|
||||
let pc = inner.pc.get().cloned();
|
||||
let req = a.clone();
|
||||
let out = tokio::task::spawn_blocking(move || pc.map(|pc| pc.act(&req))).await.ok().flatten();
|
||||
let out = out.unwrap_or_else(|| crate::pc::Outcome {
|
||||
reply: serde_json::json!({"ok": false, "error": "이 PC에서는 PC 제어를 쓸 수 없습니다"}),
|
||||
code: "unavailable".into(), activity: None });
|
||||
// The action, its result code and the time only: never a title, element or text.
|
||||
debug_log!("conversation", "pc {} {} {} ms={}", a.id, a.action.chars().take(10).collect::<String>(),
|
||||
out.code, started.elapsed().as_millis());
|
||||
if let (Some(r), Some(activity)) = (&out.activity, inner.activity.get()) {
|
||||
let by = a.user.parse().ok().map(|u| format!("{u} {}", name_of(&inner, u)));
|
||||
activity.record(&r.kind, &r.target, by, r.ok, r.note.clone());
|
||||
}
|
||||
inner.worker.pc_action_result(&a.id, out.reply);
|
||||
});
|
||||
}
|
||||
WorkerEvent::ScheduleAction(a) => {
|
||||
let result = match inner.schedules.get() {
|
||||
Some(store) => {
|
||||
|
||||
@@ -135,15 +135,7 @@ and the late "left" update cancelled the new join (seen on real Discord).
|
||||
- **Hearing**: songbird only decrypts (transport and DAVE). Its own Opus decoding is not used: in
|
||||
songbird 0.6.1 it decodes the payload without trimming what DAVE decryption left at its end, which
|
||||
garbled, repeated or dropped about a third of the frames (measured with a test tone that changes
|
||||
pitch every 20 ms). Discord names a speaker's SSRC (its Speaking op, over the websocket) only the
|
||||
first time they speak, after their first packets arrive over UDP; songbird dropped the DAVE packets
|
||||
it could not yet decrypt, which cut the first ~100 ms off the first thing each person said after a
|
||||
join (live 2026-11-18: "자비스" heard as "아비"). The app therefore uses songbird 0.6.1 with a patch
|
||||
(`app/vendor/songbird`, `JARVIS-PATCH.md`): packets of an SSRC not named yet are held, at most one
|
||||
second and 50 packets for each of at most 32 SSRCs, and processed in arrival order once it is named
|
||||
(before its next packet, or within 20 ms when none follows, as after a short first utterance);
|
||||
the Speaking event goes out first, so the core knows the speaker before their held packets come.
|
||||
The core takes each decrypted RTP packet with songbird's exact payload bounds:
|
||||
pitch every 20 ms). The core takes each decrypted RTP packet with songbird's exact payload bounds:
|
||||
- Packets go through a per-speaker reorder window in sequence-number order. A missing packet is
|
||||
waited for until two later packets have arrived (about 40 ms), then rebuilt from the next packet's
|
||||
Opus in-band FEC (the last missing frame) or concealed (any earlier ones). Late and duplicate
|
||||
@@ -179,7 +171,9 @@ and the late "left" update cancelled the new join (seen on real Discord).
|
||||
long the clip playing when it was stopped had played, timed from when it was handed to the player)
|
||||
so its voice recordings keep only what was heard (`voice_worker.spec.md`, "Recordings").
|
||||
- **Answering** (M4): `answer(user, text)` queues a request like `say` whose speech the worker makes
|
||||
from the brain's answer (`reply` with `speak`, under the request's id) and returns its id. No answer
|
||||
from the brain's answer (`reply` with `speak`, under the request's id) and returns its id. With
|
||||
`hold` it is written but does not start playing until `release(id)` (an answer that fails while held
|
||||
still ends at once and lets the queue go on). No answer
|
||||
from the brain, a refused `reply` (`brain_unavailable`) or a lost worker ends it as failed.
|
||||
`cancel(id)` stops one request (its clip if playing, the worker's unsent sentences, or the speech of
|
||||
an answer still being written) and the rest of the queue goes on. `ask_addressed(id, user, text)` and
|
||||
|
||||
@@ -22,13 +22,17 @@ pub const FEATURE_TOOLS: &[(&str, &[&str])] = &[
|
||||
("collect", &["collectedNews"]),
|
||||
];
|
||||
|
||||
/// The built-in tools of the features that are off, and `screenView` unless screen viewing is
|
||||
/// available (admin.spec.md, "Screen viewing").
|
||||
pub fn disabled_tools(s: &Settings) -> Vec<&'static str> {
|
||||
/// The built-in tools of the features that are off, `screenView` unless screen viewing is
|
||||
/// available (admin.spec.md, "Screen viewing") and `pcControl` unless PC control is (`pc`: checked
|
||||
/// with agent-desktop's hash as the services start, admin.spec.md, "PC control").
|
||||
pub fn disabled_tools(s: &Settings, pc: bool) -> Vec<&'static str> {
|
||||
let mut off = feature_tools_off(&s.features);
|
||||
if !crate::screen::available(s) {
|
||||
off.push("screenView");
|
||||
}
|
||||
if !pc {
|
||||
off.push("pcControl");
|
||||
}
|
||||
off
|
||||
}
|
||||
|
||||
@@ -120,15 +124,18 @@ mod tests {
|
||||
let mut s = Settings::default();
|
||||
s.pc_control.allowed_users = vec![1];
|
||||
let screen = if cfg!(windows) { vec![] } else { vec!["screenView"] };
|
||||
assert_eq!(disabled_tools(&s), screen, "all on by default");
|
||||
s.features = FeatureSettings { weather: false, web_search: true, meals: false, collect: false, browser: false, browser_share: false, schedule: false, notes: false, screen: true };
|
||||
assert_eq!(disabled_tools(&s, true), screen, "all on by default");
|
||||
s.features = FeatureSettings { weather: false, web_search: true, meals: false, collect: false, browser: false, browser_share: false, schedule: false, notes: false, screen: true, pc_control: true };
|
||||
let mut want = vec!["getWeather", "logMeal", "fetchMeals", "deleteMeal", "botBrowser", "schedule", "personalNotes", "collectedNews"];
|
||||
want.extend(&screen);
|
||||
assert_eq!(disabled_tools(&s), want);
|
||||
assert_eq!(disabled_tools(&s, true), want);
|
||||
// Screen viewing needs someone registered for PC control (and Windows).
|
||||
assert!(disabled_tools(&Settings::default()).contains(&"screenView"));
|
||||
assert!(disabled_tools(&Settings::default(), true).contains(&"screenView"));
|
||||
s.features.screen = false;
|
||||
assert!(disabled_tools(&s).contains(&"screenView"));
|
||||
assert!(disabled_tools(&s, true).contains(&"screenView"));
|
||||
// PC control is offered only when it is available (`pc::available` and agent-desktop).
|
||||
assert!(disabled_tools(&s, false).contains(&"pcControl"));
|
||||
assert!(!disabled_tools(&s, true).contains(&"pcControl"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -274,6 +274,11 @@ impl GameWatch {
|
||||
self.state.lock().unwrap().current().is_some()
|
||||
}
|
||||
|
||||
/// The game's name while game mode is on (PC control's anti-cheat sentence).
|
||||
pub fn name(&self) -> Option<String> {
|
||||
self.state.lock().unwrap().current().map(|d| d.name.clone())
|
||||
}
|
||||
|
||||
/// The game for the dashboard: `{name, reason}` or null.
|
||||
pub fn view(&self) -> serde_json::Value {
|
||||
serde_json::json!(self.state.lock().unwrap().current())
|
||||
|
||||
@@ -25,6 +25,7 @@ pub mod install;
|
||||
pub mod logs;
|
||||
pub mod notes;
|
||||
pub mod playback;
|
||||
pub mod pc;
|
||||
pub mod perf;
|
||||
pub mod permissions;
|
||||
pub mod placement;
|
||||
|
||||
1971
app/src/pc.rs
Normal file
1971
app/src/pc.rs
Normal file
File diff suppressed because it is too large
Load Diff
@@ -48,6 +48,8 @@ struct Request {
|
||||
failed: bool,
|
||||
played: u32,
|
||||
started: bool,
|
||||
/// Not to start playing until released (`hold`).
|
||||
held: bool,
|
||||
}
|
||||
|
||||
pub struct Playback<B: TtsBackend, S: ClipSink> {
|
||||
@@ -81,7 +83,7 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
self.counter += 1;
|
||||
let id = format!("r{}", self.counter);
|
||||
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false,
|
||||
played: 0, started: false });
|
||||
played: 0, started: false, held: false });
|
||||
crate::debug_log!("play", "say id={id} chars={} more={more} queued={}", text.chars().count(), self.queue.len());
|
||||
self.backend.tts(&id, text, more);
|
||||
id
|
||||
@@ -93,7 +95,7 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
self.counter += 1;
|
||||
let id = format!("r{}", self.counter);
|
||||
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::new(), next: 0, done: false, failed: false,
|
||||
played: 0, started: false });
|
||||
played: 0, started: false, held: false });
|
||||
crate::debug_log!("play", "answer id={id} queued={}", self.queue.len());
|
||||
id
|
||||
}
|
||||
@@ -103,7 +105,7 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
self.counter += 1;
|
||||
let id = format!("r{}", self.counter);
|
||||
self.queue.push_back(Request { id: id.clone(), chunks: BTreeMap::from([(0, pcm)]), next: 0, done: true,
|
||||
failed: false, played: 0, started: false });
|
||||
failed: false, played: 0, started: false, held: false });
|
||||
crate::debug_log!("play", "clip id={id} queued={}", self.queue.len());
|
||||
self.pump();
|
||||
id
|
||||
@@ -118,6 +120,22 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep a request that has not started from starting until `release` (the asker may go on).
|
||||
pub fn hold(&mut self, id: &str) {
|
||||
if let Some(r) = self.queue.iter_mut().find(|r| r.id == id && !r.started) {
|
||||
r.held = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// Let a held request play.
|
||||
pub fn release(&mut self, id: &str) {
|
||||
if let Some(r) = self.queue.iter_mut().find(|r| r.id == id && r.held) {
|
||||
r.held = false;
|
||||
crate::debug_log!("play", "release id={id}");
|
||||
self.pump();
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop one request: its clip if it is playing, its unsent sentences at the worker. The rest of the
|
||||
/// queue goes on.
|
||||
pub fn cancel(&mut self, id: &str) {
|
||||
@@ -207,6 +225,9 @@ impl<B: TtsBackend, S: ClipSink> Playback<B, S> {
|
||||
head.next = first;
|
||||
}
|
||||
}
|
||||
if head.held && head.chunks.contains_key(&head.next) {
|
||||
return;
|
||||
}
|
||||
if let Some(pcm) = head.chunks.remove(&head.next) {
|
||||
head.next += 1;
|
||||
head.played += 1;
|
||||
@@ -480,4 +501,28 @@ mod tests {
|
||||
assert_eq!(f.0.borrow().cancelled, vec![id.clone()]);
|
||||
assert!(p.take_events().contains(&end(&cue2, 0, true, false)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_answer_waits_for_its_release_and_a_failed_one_does_not_block() {
|
||||
let (mut p, f) = setup();
|
||||
let id = p.adopt();
|
||||
p.hold(&id);
|
||||
p.audio(&id, 0, b("a0"));
|
||||
p.done(&done(&id, false));
|
||||
assert!(f.0.borrow().played.is_empty(), "written, but held");
|
||||
assert!(p.take_events().is_empty());
|
||||
p.release(&id);
|
||||
assert_eq!(f.0.borrow().played, vec!["a0"]);
|
||||
assert_eq!(p.take_events(), vec![PlaybackEvent::Start(id.clone())]);
|
||||
// An answer that failed while held ends at once, and the next request plays.
|
||||
p.clip_ended();
|
||||
p.take_events();
|
||||
let bad = p.adopt();
|
||||
p.hold(&bad);
|
||||
let next = p.say("b", false);
|
||||
p.audio(&next, 0, b("b0"));
|
||||
p.fail(&bad);
|
||||
assert_eq!(f.0.borrow().played, vec!["a0", "b0"]);
|
||||
assert_eq!(p.take_events()[0], end(&bad, 0, false, true));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -254,6 +254,21 @@ impl Policy {
|
||||
}
|
||||
}
|
||||
|
||||
/// The request is taken back before its answer was heard: its speaker went on talking and it is
|
||||
/// joined to what they said next (conversation.spec.md, "Going on"). Not a missed request. When
|
||||
/// it is the running answer it is stopped; nothing else starts until the joined request comes.
|
||||
pub fn withdraw(&mut self, id: u64) -> Vec<Action> {
|
||||
if self.current.as_ref().is_some_and(|r| r.id == id) {
|
||||
self.current = None;
|
||||
return vec![Action::StopAnswer(id)];
|
||||
}
|
||||
self.waiting.retain(|r| r.id != id);
|
||||
if self.ranking.as_ref().is_some_and(|r| r.ids.contains(&id)) {
|
||||
self.ranking = None;
|
||||
}
|
||||
vec![]
|
||||
}
|
||||
|
||||
/// A stop command: stop the answer and drop everything waiting.
|
||||
pub fn stop(&mut self) -> Vec<Action> {
|
||||
let mut out = vec![];
|
||||
|
||||
@@ -96,6 +96,8 @@ pub enum Target {
|
||||
Screen,
|
||||
Monitor(u32),
|
||||
Window(String),
|
||||
/// Someone's Discord screen share, in the window Discord pops it out to.
|
||||
Share,
|
||||
}
|
||||
|
||||
/// The activity log's target: never a title.
|
||||
@@ -104,6 +106,7 @@ pub fn activity_target(t: &Target) -> String {
|
||||
Target::Screen => "전체 화면".into(),
|
||||
Target::Monitor(n) => format!("모니터 {n}"),
|
||||
Target::Window(_) => "창".into(),
|
||||
Target::Share => "화면공유".into(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -120,6 +123,7 @@ pub fn target_of(a: &ScreenAction) -> Result<(Target, Detail), String> {
|
||||
let w = a.window.as_deref().map(str::trim).filter(|w| !w.is_empty()).ok_or("창 이름이 필요합니다")?;
|
||||
Target::Window(w.chars().take(200).collect())
|
||||
}
|
||||
Some("share") => Target::Share,
|
||||
_ => Target::Screen,
|
||||
};
|
||||
Ok((target, detail))
|
||||
@@ -169,6 +173,55 @@ pub fn pick_window<'a>(s: &'a Settings, all: &'a [Window], text: &str) -> Option
|
||||
}
|
||||
|
||||
/// Whether nothing shows (protected video, exclusive full screen): every pixel near black.
|
||||
/// Discord's programs (stable, PTB, Canary).
|
||||
const DISCORD: [&str; 3] = ["discord.exe", "discordptb.exe", "discordcanary.exe"];
|
||||
|
||||
/// What a share capture gives while Discord has no popped-out screen share window: the core then
|
||||
/// asks for one and waits (admin.spec.md, "Screen viewing").
|
||||
pub const NO_SHARE_WINDOW: &str = "분리된 화면공유 창이 없습니다";
|
||||
|
||||
/// The window Discord pops a screen share out to: a Discord window that is not its main one (whose
|
||||
/// title ends with "Discord"). Its title is the voice channel's name, so the one naming Jarvis's
|
||||
/// voice channel goes first, else the one nearest the front.
|
||||
pub fn share_window<'a>(s: &'a Settings, all: &'a [Window], channel: Option<&str>) -> Option<&'a Window> {
|
||||
let shares: Vec<&Window> = listable(s, all)
|
||||
.filter(|w| DISCORD.contains(&w.program.as_str()) && !w.title.trim().to_lowercase().ends_with("discord"))
|
||||
.collect();
|
||||
let channel = channel.map(|c| c.trim().to_lowercase()).filter(|c| !c.is_empty());
|
||||
channel.and_then(|c| shares.iter().find(|w| w.title.to_lowercase().contains(&c)).copied())
|
||||
.or_else(|| shares.first().copied())
|
||||
}
|
||||
|
||||
/// Letters and digits in lower case, so a spoken name and a display name compare.
|
||||
fn name_key(name: &str) -> String {
|
||||
name.chars().filter(|c| c.is_alphanumeric()).flat_map(char::to_lowercase).collect()
|
||||
}
|
||||
|
||||
/// Whose share the popped-out window should show, from the people sharing their screen in Jarvis's
|
||||
/// voice channel (ID, display name): the person asked about (their name inside what was said, or
|
||||
/// what was said inside their name), or with no name anyone sharing (`None` when several do).
|
||||
pub fn share_streamer(streaming: &[(u64, String)], person: Option<&str>) -> Result<Option<String>, String> {
|
||||
let person = person.map(str::trim).filter(|p| !name_key(p).is_empty());
|
||||
let Some(person) = person else {
|
||||
return match streaming {
|
||||
[] => Err("지금 음성 채널에서 화면공유 중인 사람이 없습니다".into()),
|
||||
[(_, name)] => Ok(Some(name.clone())),
|
||||
_ => Ok(None),
|
||||
};
|
||||
};
|
||||
let key = name_key(person);
|
||||
streaming.iter()
|
||||
.find(|(_, name)| { let n = name_key(name); !n.is_empty() && (key.contains(&n) || n.contains(&key)) })
|
||||
.map(|(_, name)| Some(name.clone()))
|
||||
.ok_or_else(|| format!("{}님은 지금 화면공유 중이 아닙니다", person.chars().take(40).collect::<String>()))
|
||||
}
|
||||
|
||||
/// What Jarvis says in the voice channel while it waits for the share to be popped out.
|
||||
pub fn share_prompt(name: Option<&str>) -> String {
|
||||
let whose = name.map(|n| format!("{n}님 화면공유를")).unwrap_or_else(|| "화면공유를".into());
|
||||
format!("디스코드에서 {whose} 시청하고 창 분리 버튼을 눌러 주세요.")
|
||||
}
|
||||
|
||||
pub fn all_black(img: &Image) -> bool {
|
||||
img.bgra.chunks_exact(4).all(|p| p[0] <= 8 && p[1] <= 8 && p[2] <= 8)
|
||||
}
|
||||
@@ -330,6 +383,16 @@ pub fn act(s: &Settings, screen: &dyn Screen, a: &ScreenAction) -> Result<Value,
|
||||
let black = all_black(&img);
|
||||
(img, black, format!("창 {}", w.program.strip_suffix(".exe").unwrap_or(&w.program)))
|
||||
}
|
||||
Target::Share => {
|
||||
let all = screen.windows()?;
|
||||
let w = share_window(s, &all, a.voice_channel.as_deref()).ok_or(NO_SHARE_WINDOW)?;
|
||||
if w.minimised {
|
||||
return Err("화면공유 창이 최소화되어 있어 볼 수 없습니다. 창을 다시 열어 주세요".into());
|
||||
}
|
||||
let img = screen.capture_window(w)?;
|
||||
let black = all_black(&img);
|
||||
(img, black, activity_target(&target))
|
||||
}
|
||||
_ => {
|
||||
let ms = screen.monitors()?;
|
||||
let chosen: Vec<Monitor> = match target {
|
||||
@@ -813,6 +876,73 @@ mod tests {
|
||||
assert!(v.get("images").is_none());
|
||||
}
|
||||
|
||||
fn with_share(f: &mut Fake, popout: Window) {
|
||||
f.windows.insert(1, win("discord.exe", "#일반 | 우리 서버 - Discord", Rect { x: 0, y: 0, w: 1200, h: 800 }));
|
||||
f.windows.insert(0, popout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_screen_share_is_seen_in_discords_popped_out_window_never_its_main_one() {
|
||||
let s = settings();
|
||||
let mut f = fake();
|
||||
let mut a = action("capture", OWNER, json!({"target": "share", "person": "철수"}));
|
||||
a.voice_channel = Some("게임방".into());
|
||||
// Only the main window: nothing popped out yet.
|
||||
f.windows.insert(1, win("discord.exe", "#일반 | 우리 서버 - Discord", Rect { x: 0, y: 0, w: 1200, h: 800 }));
|
||||
assert_eq!(act(&s, &f, &a).unwrap_err(), NO_SHARE_WINDOW);
|
||||
assert!(f.captured.borrow().is_empty());
|
||||
|
||||
let mut f = fake();
|
||||
with_share(&mut f, win("discord.exe", "게임방", Rect { x: 0, y: 0, w: 960, h: 540 }));
|
||||
let v = act(&s, &f, &a).unwrap();
|
||||
assert_eq!(*f.captured.borrow(), vec!["window discord.exe"]);
|
||||
assert_eq!(v["item"]["what"], "화면공유");
|
||||
assert_eq!(activity_target(&target_of(&a).unwrap().0), "화면공유");
|
||||
|
||||
// A popped-out window that is minimised is asked to be opened again.
|
||||
let mut f = fake();
|
||||
with_share(&mut f, Window { minimised: true, ..win("discord.exe", "게임방", Rect { x: 0, y: 0, w: 960, h: 540 }) });
|
||||
assert!(act(&s, &f, &a).unwrap_err().contains("최소화"));
|
||||
|
||||
// Discord on the exclusion list is never seen, popped out or not.
|
||||
let mut s = settings();
|
||||
s.pc_control.screen_exclude.push("Discord".into());
|
||||
let mut f = fake();
|
||||
with_share(&mut f, win("discord.exe", "게임방", Rect { x: 0, y: 0, w: 960, h: 540 }));
|
||||
assert_eq!(act(&s, &f, &a).unwrap_err(), NO_SHARE_WINDOW);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_popped_out_window_naming_jarvis_voice_channel_goes_first() {
|
||||
let s = settings();
|
||||
let all = vec![
|
||||
win("discordptb.exe", "다른 방", Rect { x: 0, y: 0, w: 10, h: 10 }),
|
||||
win("discordptb.exe", "게임방", Rect { x: 0, y: 0, w: 10, h: 10 }),
|
||||
win("discordptb.exe", "Discord", Rect { x: 0, y: 0, w: 10, h: 10 }),
|
||||
];
|
||||
assert_eq!(share_window(&s, &all, Some("게임방")).unwrap().title, "게임방");
|
||||
assert_eq!(share_window(&s, &all, None).unwrap().title, "다른 방", "else the one nearest the front");
|
||||
assert!(share_window(&s, &all[2..], Some("게임방")).is_none(), "the main window is never the share");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_share_is_seen_only_while_the_person_asked_about_shares_their_screen() {
|
||||
let streaming = vec![(1, "철수".to_owned()), (2, "Younghee".to_owned())];
|
||||
assert_eq!(share_streamer(&streaming, Some("철수님")), Ok(Some("철수".to_owned())), "a spoken suffix still matches");
|
||||
assert_eq!(share_streamer(&streaming, Some("younghee")), Ok(Some("Younghee".to_owned())));
|
||||
assert_eq!(share_streamer(&streaming, Some("민수")).unwrap_err(), "민수님은 지금 화면공유 중이 아닙니다");
|
||||
assert_eq!(share_streamer(&streaming, None), Ok(None), "no name: whatever the window shows");
|
||||
assert_eq!(share_streamer(&streaming[..1], Some(" ")), Ok(Some("철수".to_owned())));
|
||||
assert!(share_streamer(&[], None).unwrap_err().contains("화면공유 중인 사람이 없습니다"));
|
||||
assert!(share_streamer(&[], Some("철수")).unwrap_err().contains("철수님은"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_share_prompt_names_the_person_when_it_is_known() {
|
||||
assert!(share_prompt(Some("철수")).contains("철수님 화면공유"));
|
||||
assert!(share_prompt(None).contains("화면공유를"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scaling_averages_the_area() {
|
||||
let mut img = solid(4, 2, 0);
|
||||
|
||||
@@ -31,6 +31,8 @@ pub struct Services {
|
||||
pub schedules: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
|
||||
/// Announces new collected items (admin.spec.md, "News notifications").
|
||||
pub news: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
|
||||
/// PC control's emergency-stop hotkey while PC control is available (admin.spec.md, "PC control").
|
||||
pub pc_hotkey: std::sync::Mutex<Option<crate::pc::Hotkey>>,
|
||||
}
|
||||
|
||||
/// The bot browser of a run (admin.spec.md, "Bot browser").
|
||||
@@ -82,6 +84,9 @@ impl Services {
|
||||
n.abort();
|
||||
let _ = n.await;
|
||||
}
|
||||
if let Some(h) = self.pc_hotkey.lock().unwrap().take() {
|
||||
h.end();
|
||||
}
|
||||
self.game.stop();
|
||||
self.core.close().await;
|
||||
self.worker.close();
|
||||
@@ -155,8 +160,10 @@ pub async fn start_services(data_dir: &Path, llm: Arc<Llm>) -> Result<Services,
|
||||
hub.set_secrets_in_use(secrets_in_use(&token, &mcp_env, kma_key.as_deref()));
|
||||
}
|
||||
|
||||
// PC control: agent-desktop's hash is checked once per start (admin.spec.md, "PC control").
|
||||
let pc_exe = crate::pc::check(&settings);
|
||||
let (base_url, model) = llm.start(&settings).await?;
|
||||
let brain_config = match crate::strata::write_brain_config(data_dir, &base_url, &model, &settings) {
|
||||
let brain_config = match crate::strata::write_brain_config(data_dir, &base_url, &model, &settings, pc_exe.is_some()) {
|
||||
Ok(f) => f,
|
||||
Err(e) => return Err(after_llm_stop(&llm, format!("could not write the brain settings: {e}")).await.into()),
|
||||
};
|
||||
@@ -214,7 +221,19 @@ pub async fn start_services(data_dir: &Path, llm: Arc<Llm>) -> Result<Services,
|
||||
let chat_calls = conversation::ChatCalls::from_settings(&settings);
|
||||
let schedule_channel = settings.schedule_channel;
|
||||
let news_settings = settings.clone();
|
||||
let pc = pc_exe.as_ref().map(|exe| {
|
||||
let g = game.clone();
|
||||
crate::pc::Pc::new(settings.clone(), Box::new(crate::pc::Cli::new(exe, data_dir)), crate::pc::emergency(),
|
||||
Box::new(move || g.name()))
|
||||
});
|
||||
let stop_hotkey = settings.pc_control.stop_hotkey.clone();
|
||||
let (core, core_events) = DiscordCore::new(settings, token, worker.clone(), worker_events);
|
||||
let pc_hotkey = std::sync::Mutex::new(None);
|
||||
if let Some(pc) = pc {
|
||||
core.set_pc(Arc::new(pc));
|
||||
*pc_hotkey.lock().unwrap() = crate::pc::Hotkey::start(&stop_hotkey, crate::pc::emergency());
|
||||
crate::info_log!("conversation", "PC control available (agent-desktop {})", crate::pc::AGENT_DESKTOP_VERSION);
|
||||
}
|
||||
let schedule_store = Arc::new(crate::schedule::Store::new(data_dir));
|
||||
core.set_schedules(schedule_store.clone());
|
||||
let activity = Arc::new(crate::activity::Activity::new(data_dir));
|
||||
@@ -237,7 +256,7 @@ pub async fn start_services(data_dir: &Path, llm: Arc<Llm>) -> Result<Services,
|
||||
Arc::new(core.clone()), activity))));
|
||||
let on = { let g = game.clone(); Arc::new(move || g.on()) };
|
||||
let perf = std::sync::Mutex::new(Some(tokio::spawn(perf_loop(perf_settings, llm.clone(), Arc::new(worker.clone()), on, RETRY_AFTER))));
|
||||
Ok(Services { core, events, worker, worker_process, missed, llm, browser, game, perf, schedules, news })
|
||||
Ok(Services { core, events, worker, worker_process, missed, llm, browser, game, perf, schedules, news, pc_hotkey })
|
||||
}
|
||||
|
||||
/// The voice worker writes its recordings where this names (voice_worker.spec.md, "Recordings").
|
||||
|
||||
@@ -106,6 +106,8 @@ pub struct FeatureSettings {
|
||||
pub notes: bool,
|
||||
/// Screen viewing (admin.spec.md, "Screen viewing").
|
||||
pub screen: bool,
|
||||
/// Operating programs through agent-desktop (admin.spec.md, "PC control").
|
||||
pub pc_control: bool,
|
||||
}
|
||||
|
||||
/// Game detection (admin.spec.md, "Game detection"): on by default; the person's own games and
|
||||
@@ -187,6 +189,11 @@ pub struct PcControlSettings {
|
||||
pub run: Level,
|
||||
/// Programs never seen on the screen (file names, ".exe" may be left out).
|
||||
pub screen_exclude: Vec<String>,
|
||||
/// Programs with an anti-cheat the person added (file names, ".exe" may be left out): while one
|
||||
/// runs, nothing but reading is done (admin.spec.md, "PC control").
|
||||
pub anti_cheat_games: Vec<String>,
|
||||
/// The emergency stop's global hotkey (`Ctrl+Shift+F12`).
|
||||
pub stop_hotkey: String,
|
||||
}
|
||||
|
||||
/// An MCP server the brain starts (stdio). The values of `env` are in the secrets, never here.
|
||||
@@ -247,6 +254,12 @@ impl std::error::Error for SettingsError {}
|
||||
pub const DEFAULT_JOIN_NOTICE: &str =
|
||||
"자비스가 이 음성 채널에 들어왔습니다. 이 채널의 대화는 자비스가 받아 적습니다.";
|
||||
|
||||
/// The default wake words: Jarvis is called like a secretary, by the word and its usual call forms
|
||||
/// (the longest one heard is the call, `calls.rs`).
|
||||
pub fn default_wake_words() -> Vec<String> {
|
||||
["비서", "비서야", "비서님", "김비서", "김비서야", "김비서님"].map(String::from).to_vec()
|
||||
}
|
||||
|
||||
impl Default for Settings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
@@ -261,7 +274,7 @@ impl Default for Settings {
|
||||
listen_to_bots: false,
|
||||
record: false,
|
||||
join_notice: DEFAULT_JOIN_NOTICE.into(),
|
||||
wake_words: vec!["자비스".into()],
|
||||
wake_words: default_wake_words(),
|
||||
stt: Map::new(),
|
||||
tts_speed: 1.5,
|
||||
},
|
||||
@@ -276,14 +289,15 @@ impl Default for Settings {
|
||||
response: PolicySettings::default(),
|
||||
memory: Map::new(),
|
||||
llm: LlmSettings { url: None, port: 8080, context: 32768 },
|
||||
features: FeatureSettings { weather: true, web_search: true, meals: true, collect: true, browser: true, browser_share: false, schedule: true, notes: true, screen: true },
|
||||
features: FeatureSettings { weather: true, web_search: true, meals: true, collect: true, browser: true, browser_share: false, schedule: true, notes: true, screen: true, pc_control: true },
|
||||
mcp_servers: vec![],
|
||||
collect: CollectSettings { keep_days: 30, targets: vec![], notify_channel: None, digest_hour: 9 },
|
||||
game: GameSettings { detect: true, games: vec![], not_games: vec![], other_room_gb: 1.0, reserve_gb: 8.0, revert_seconds: 30, pause_in_game: false },
|
||||
schedule_channel: None,
|
||||
pc_control: PcControlSettings { allowed_users: vec![], view: Level::Auto, click: Level::Ask,
|
||||
type_text: Level::Ask, run: Level::Ask,
|
||||
screen_exclude: ["1Password", "Bitwarden", "KeePass", "KeePassXC"].map(String::from).into() },
|
||||
screen_exclude: ["1Password", "Bitwarden", "KeePass", "KeePassXC"].map(String::from).into(),
|
||||
anti_cheat_games: vec![], stop_hotkey: crate::pc::DEFAULT_STOP_HOTKEY.into() },
|
||||
close_to_tray: false,
|
||||
location: Some(DEFAULT_LOCATION.into()),
|
||||
}
|
||||
@@ -439,6 +453,8 @@ fn parse_call(raw: &Map<String, Value>, wake_words: &[String], d: &CallSettings)
|
||||
designated_no_wake_word: get_bool(call, "designatedNoWakeWord", "call", d.designated_no_wake_word)?,
|
||||
stop_words,
|
||||
follow_up: get_ms(call, "followUpMs", "call", d.follow_up, 5000)?,
|
||||
continue_window: get_ms(call, "continueMs", "call", d.continue_window, 15000)?,
|
||||
answer_wait: get_ms(call, "answerWaitMs", "call", d.answer_wait, 10000)?,
|
||||
earcon: get_bool(call, "earcon", "call", d.earcon)?,
|
||||
},
|
||||
chat_wake_words,
|
||||
@@ -627,6 +643,7 @@ pub fn parse_settings(raw: &Value) -> Result<Settings, SettingsError> {
|
||||
schedule: get_bool(f, "schedule", "features", d.features.schedule)?,
|
||||
notes: get_bool(f, "notes", "features", d.features.notes)?,
|
||||
screen: get_bool(f, "screen", "features", d.features.screen)?,
|
||||
pc_control: get_bool(f, "pcControl", "features", d.features.pc_control)?,
|
||||
}
|
||||
},
|
||||
mcp_servers: parse_mcp_servers(raw)?,
|
||||
@@ -660,20 +677,27 @@ fn parse_pc_control(raw: &Map<String, Value>, d: &PcControlSettings) -> Result<P
|
||||
Some(_) => err(format!("pcControl.{key} must be auto, ask or never")),
|
||||
}
|
||||
};
|
||||
let programs = |key: &str, default: &[String]| -> Result<Vec<String>, SettingsError> {
|
||||
let v: Vec<String> = get_str_list(p, key, "pcControl", default)?.into_iter()
|
||||
.map(|e| e.trim().to_owned()).filter(|e| !e.is_empty()).collect();
|
||||
if v.len() > 100 || v.iter().any(|e| e.chars().count() > 100 || e.contains(['\\', '/'])) {
|
||||
return err(format!("pcControl.{key} must be at most 100 program names (no folders)"));
|
||||
}
|
||||
Ok(v)
|
||||
};
|
||||
let stop_hotkey = get_str(p, "stopHotkey", "pcControl", &d.stop_hotkey)?.trim().to_owned();
|
||||
if crate::pc::parse_hotkey(&stop_hotkey).is_none() {
|
||||
return err("pcControl.stopHotkey must be modifiers and a key, such as Ctrl+Shift+F12");
|
||||
}
|
||||
Ok(PcControlSettings {
|
||||
allowed_users: snowflake_list(p, "allowedUsers", "pcControl")?,
|
||||
view: level("view", d.view)?,
|
||||
click: level("click", d.click)?,
|
||||
type_text: level("type", d.type_text)?,
|
||||
run: level("run", d.run)?,
|
||||
screen_exclude: {
|
||||
let v: Vec<String> = get_str_list(p, "screenExclude", "pcControl", &d.screen_exclude)?.into_iter()
|
||||
.map(|e| e.trim().to_owned()).filter(|e| !e.is_empty()).collect();
|
||||
if v.len() > 100 || v.iter().any(|e| e.chars().count() > 100 || e.contains(['\\', '/'])) {
|
||||
return err("pcControl.screenExclude must be at most 100 program names (no folders)");
|
||||
}
|
||||
v
|
||||
},
|
||||
screen_exclude: programs("screenExclude", &d.screen_exclude)?,
|
||||
anti_cheat_games: programs("antiCheatGames", &d.anti_cheat_games)?,
|
||||
stop_hotkey,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1054,13 +1078,13 @@ mod tests {
|
||||
#[test]
|
||||
fn features_and_mcp_servers_are_read_and_checked() {
|
||||
let s = parse_settings(&json!({}));
|
||||
assert_eq!(s.unwrap().features, FeatureSettings { weather: true, web_search: true, meals: true, collect: true, browser: true, browser_share: false, schedule: true, notes: true, screen: true });
|
||||
assert_eq!(s.unwrap().features, FeatureSettings { weather: true, web_search: true, meals: true, collect: true, browser: true, browser_share: false, schedule: true, notes: true, screen: true, pc_control: true });
|
||||
let s = parse_settings(&json!({
|
||||
"features": {"weather": false, "meals": false, "browserShare": true, "schedule": false, "notes": false, "screen": false},
|
||||
"features": {"weather": false, "meals": false, "browserShare": true, "schedule": false, "notes": false, "screen": false, "pcControl": false},
|
||||
"mcpServers": [{"name": "brave", "command": "npx", "args": ["-y", "brave"], "env": ["BRAVE_API_KEY", " "]},
|
||||
{"name": "time-2", "command": "uvx", "enabled": false}]
|
||||
})).unwrap();
|
||||
assert_eq!(s.features, FeatureSettings { weather: false, web_search: true, meals: false, collect: true, browser: true, browser_share: true, schedule: false, notes: false, screen: false });
|
||||
assert_eq!(s.features, FeatureSettings { weather: false, web_search: true, meals: false, collect: true, browser: true, browser_share: true, schedule: false, notes: false, screen: false, pc_control: false });
|
||||
assert_eq!(s.mcp_servers, vec![
|
||||
McpServerSettings { name: "brave".into(), command: "npx".into(), args: vec!["-y".into(), "brave".into()],
|
||||
env: vec!["BRAVE_API_KEY".into()], enabled: true },
|
||||
@@ -1071,13 +1095,23 @@ mod tests {
|
||||
assert!(pc.allowed_users.is_empty());
|
||||
assert_eq!((pc.view, pc.click, pc.type_text, pc.run), (Level::Auto, Level::Ask, Level::Ask, Level::Ask));
|
||||
assert!(pc.screen_exclude.contains(&"KeePassXC".to_owned()), "password managers are never seen by default");
|
||||
assert!(pc.anti_cheat_games.is_empty());
|
||||
assert_eq!(pc.stop_hotkey, "Ctrl+Shift+F12");
|
||||
let pc = parse_settings(&json!({"pcControl": {"allowedUsers": ["123456789012345678"], "view": "never",
|
||||
"click": "auto", "type": "never", "run": "auto", "screenExclude": [" bank.exe ", ""]}})).unwrap().pc_control;
|
||||
"click": "auto", "type": "never", "run": "auto", "screenExclude": [" bank.exe ", ""],
|
||||
"antiCheatGames": ["MyShooter"], "stopHotkey": "Ctrl+Shift+Nope"}})).unwrap_err();
|
||||
assert!(pc.0.contains("pcControl.stopHotkey"), "{pc}");
|
||||
let pc = parse_settings(&json!({"pcControl": {"allowedUsers": ["123456789012345678"], "view": "never",
|
||||
"click": "auto", "type": "never", "run": "auto", "screenExclude": [" bank.exe ", ""],
|
||||
"antiCheatGames": ["MyShooter"], "stopHotkey": " ctrl+alt+s "}})).unwrap().pc_control;
|
||||
assert_eq!(pc.allowed_users, vec![123456789012345678]);
|
||||
assert_eq!(pc.screen_exclude, vec!["bank.exe".to_owned()]);
|
||||
assert_eq!(pc.anti_cheat_games, vec!["MyShooter".to_owned()]);
|
||||
assert_eq!(pc.stop_hotkey, "ctrl+alt+s");
|
||||
assert_eq!((pc.view, pc.click, pc.type_text, pc.run), (Level::Never, Level::Auto, Level::Never, Level::Auto));
|
||||
for bad in [json!({"pcControl": {"click": "yes"}}), json!({"pcControl": {"allowedUsers": ["1"]}}),
|
||||
json!({"pcControl": {"screenExclude": ["C:\\bank\\bank.exe"]}}), json!({"pcControl": {"screenExclude": "x"}})] {
|
||||
json!({"pcControl": {"screenExclude": ["C:\\bank\\bank.exe"]}}), json!({"pcControl": {"screenExclude": "x"}}),
|
||||
json!({"pcControl": {"antiCheatGames": ["D:\\games\\x.exe"]}}), json!({"pcControl": {"stopHotkey": "F12"}})] {
|
||||
assert!(parse_settings(&bad).unwrap_err().0.contains("pcControl."), "{bad}");
|
||||
}
|
||||
// A schedule channel must be a registered one that Jarvis may send in.
|
||||
|
||||
@@ -15,10 +15,13 @@ use crate::settings::{parse_settings, DEFAULT_JOIN_NOTICE};
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Screen {
|
||||
Bot,
|
||||
Channels,
|
||||
VoiceChannel,
|
||||
Call,
|
||||
Response,
|
||||
Chat,
|
||||
Memory,
|
||||
Model,
|
||||
Voice,
|
||||
Llm,
|
||||
Features,
|
||||
Collect,
|
||||
Game,
|
||||
@@ -78,33 +81,35 @@ pub fn fields() -> Vec<Field> {
|
||||
vec![
|
||||
f("discord.commandName", Bot, Text { max: 32, non_empty: true }, json!("자비스"), "슬래시 명령 이름"),
|
||||
f("discord.ephemeralReplies", Bot, Bool, json!(true), "명령 응답은 부른 사람만 보기"),
|
||||
f("discord.guildId", Channels, Id, Value::Null, "서버"),
|
||||
f("discord.voiceChannelId", Channels, Id, Value::Null, "자동으로 들어갈 음성 채널"),
|
||||
f("voice.maxSpeakers", Channels, Int { min: 1, max: 10 }, json!(10), "동시에 받아 적을 최대 인원"),
|
||||
f("voice.listenToBots", Channels, Bool, json!(false), "다른 봇의 말도 받아 적기"),
|
||||
f("voice.wakeWords", Channels, TextList { max_items: 20, nullable: false }, json!(["자비스"]), "호출어"),
|
||||
f("call.mode", Channels, Choice { values: &["wakeWord", "addressed", "everything"] }, json!("wakeWord"),
|
||||
f("discord.guildId", VoiceChannel, Id, Value::Null, "서버"),
|
||||
f("discord.voiceChannelId", VoiceChannel, Id, Value::Null, "자동으로 들어갈 음성 채널"),
|
||||
f("voice.maxSpeakers", VoiceChannel, Int { min: 1, max: 10 }, json!(10), "동시에 받아 적을 최대 인원"),
|
||||
f("voice.listenToBots", VoiceChannel, Bool, json!(false), "다른 봇의 말도 받아 적기"),
|
||||
f("voice.wakeWords", Call, TextList { max_items: 20, nullable: false }, json!(crate::settings::default_wake_words()), "호출어"),
|
||||
f("call.mode", Call, Choice { values: &["wakeWord", "addressed", "everything"] }, json!("wakeWord"),
|
||||
"호출 방식"),
|
||||
f("call.similarity", Channels, Number { min: 0.0, max: 0.5 }, json!(0.2), "비슷한 발음 허용 정도"),
|
||||
f("call.designated", Channels, IdList, json!([]), "지정한 사람"),
|
||||
f("call.designatedOnly", Channels, Bool, json!(false), "지정한 사람에게만 응답"),
|
||||
f("call.designatedNoWakeWord", Channels, Bool, json!(false), "지정한 사람은 호출어 없이 대답"),
|
||||
f("call.stopWords", Channels, TextList { max_items: 20, nullable: false },
|
||||
f("call.similarity", Call, Number { min: 0.0, max: 0.5 }, json!(0.2), "비슷한 발음 허용 정도"),
|
||||
f("call.designated", Call, IdList, json!([]), "지정한 사람"),
|
||||
f("call.designatedOnly", Call, Bool, json!(false), "지정한 사람에게만 응답"),
|
||||
f("call.designatedNoWakeWord", Call, Bool, json!(false), "지정한 사람은 호출어 없이 대답"),
|
||||
f("call.stopWords", Call, TextList { max_items: 20, nullable: false },
|
||||
json!(["멈춰", "그만", "정지", "stop"]), "정지 명령"),
|
||||
f("call.followUpMs", Channels, Int { min: 0, max: 5000 }, json!(1500), "호출어 뒤 이어 말하기 대기(ms)"),
|
||||
f("call.earcon", Channels, Bool, json!(true), "부르면 효과음"),
|
||||
f("response.policy", Channels, Choice { values: &["first", "inOrder", "latest", "important"] },
|
||||
f("call.followUpMs", Call, Int { min: 0, max: 5000 }, json!(1500), "호출어 뒤 이어 말하기 대기(ms)"),
|
||||
f("call.continueMs", Call, Int { min: 0, max: 15000 }, json!(5000), "끊어 말한 문장 합치기(ms)"),
|
||||
f("call.answerWaitMs", Call, Int { min: 0, max: 10000 }, json!(3000), "말이 끝나고 답하기 전 기다림(ms)"),
|
||||
f("call.earcon", Call, Bool, json!(true), "부르면 효과음"),
|
||||
f("response.policy", Response, Choice { values: &["first", "inOrder", "latest", "important"] },
|
||||
json!("latest"), "동시 대화 응답 정책"),
|
||||
f("response.queueMax", Channels, Int { min: 1, max: 20 }, json!(5), "기다리는 요청 최대 개수"),
|
||||
f("response.collectMs", Channels, Int { min: 0, max: 3000 }, json!(700), "중요도 비교 대기(ms)"),
|
||||
f("response.settleMs", Channels, Int { min: 0, max: 3000 }, json!(800), "먼저 끝난 말 전사 대기(ms)"),
|
||||
f("response.priorities", Channels, IdNumberMap, json!({}), "사람별 우선순위"),
|
||||
f("response.interrupt", Channels, Bool, json!(true),
|
||||
f("response.queueMax", Response, Int { min: 1, max: 20 }, json!(5), "기다리는 요청 최대 개수"),
|
||||
f("response.collectMs", Response, Int { min: 0, max: 3000 }, json!(700), "중요도 비교 대기(ms)"),
|
||||
f("response.settleMs", Response, Int { min: 0, max: 3000 }, json!(800), "먼저 끝난 말 전사 대기(ms)"),
|
||||
f("response.priorities", Response, IdNumberMap, json!({}), "사람별 우선순위"),
|
||||
f("response.interrupt", Response, Bool, json!(true),
|
||||
"말하는 중에 새 요청이 오면 하던 말 멈추기(가장 최근 것, 중요한 것 먼저)"),
|
||||
f("voice.joinNotice", Channels, Text { max: 2000, non_empty: true }, json!(DEFAULT_JOIN_NOTICE),
|
||||
f("voice.joinNotice", VoiceChannel, Text { max: 2000, non_empty: true }, json!(DEFAULT_JOIN_NOTICE),
|
||||
"입장 시 전사 고지 문구"),
|
||||
f("chatChannels", Chat, ChatChannels, json!([]), "채팅 채널"),
|
||||
f("call.chatWakeWords", Chat, TextList { max_items: 20, nullable: true }, Value::Null,
|
||||
f("call.chatWakeWords", Call, TextList { max_items: 20, nullable: true }, Value::Null,
|
||||
"채팅 호출어(비우면 음성과 같음)"),
|
||||
// The worker's memory settings and their defaults (voice_worker.spec.md, "Memory settings").
|
||||
f("memory.chat", Memory, Bool, json!(true), "등록한 채팅 채널 참고"),
|
||||
@@ -116,21 +121,21 @@ pub fn fields() -> Vec<Field> {
|
||||
f("memory.recent_records", Memory, Int { min: 0, max: 200 }, json!(20), "최근 대화 원문 개수"),
|
||||
f("memory.matches", Memory, Int { min: 0, max: 20 }, json!(5), "과거 기록 가져올 개수"),
|
||||
f("memory.budget_tokens", Memory, Number { min: 0.0, max: 100000.0 }, json!(4000), "참고 분량 상한(토큰)"),
|
||||
f("voice.record", Memory, Bool, json!(false),
|
||||
f("voice.record", VoiceChannel, Bool, json!(false),
|
||||
"목소리 녹음 저장(사람 말과 자비스 말, 분석용, 7일 보관, 다음 시작부터)"),
|
||||
// The worker's STT settings and their defaults (voice_worker.spec.md, "STT").
|
||||
f("voice.stt.end_silence_ms", Model, Int { min: 100, max: 2000 }, json!(300), "말 끝 판정 시간(ms)"),
|
||||
f("voice.stt.partials", Model, Bool, json!(true), "중간 전사"),
|
||||
f("voice.stt.partial_interval_ms", Model, Int { min: 100, max: 5000 }, json!(400), "중간 전사 간격(ms)"),
|
||||
f("voice.stt.final_beam", Model, Int { min: 1, max: 5 }, json!(2), "최종 전사 정확도(빔)"),
|
||||
f("voice.stt.language", Model, LanguageCode, json!("ko"), "받아 적을 언어(비우면 자동)"),
|
||||
f("voice.stt.end_silence_ms", Voice, Int { min: 100, max: 2000 }, json!(300), "말 끝 판정 시간(ms)"),
|
||||
f("voice.stt.partials", Voice, Bool, json!(true), "중간 전사"),
|
||||
f("voice.stt.partial_interval_ms", Voice, Int { min: 100, max: 5000 }, json!(400), "중간 전사 간격(ms)"),
|
||||
f("voice.stt.final_beam", Voice, Int { min: 1, max: 5 }, json!(2), "최종 전사 정확도(빔)"),
|
||||
f("voice.stt.language", Voice, LanguageCode, json!("ko"), "받아 적을 언어(비우면 자동)"),
|
||||
// MeloTTS's Korean model has one voice (speaker "KR"), so only the speed is a choice.
|
||||
f("voice.tts.speed", Model, Number { min: 0.5, max: 2.0 }, json!(1.5), "말하기 속도(1.0 보통, 기본 1.5, 저장하면 바로 적용)"),
|
||||
f("voice.tts.speed", Voice, Number { min: 0.5, max: 2.0 }, json!(1.5), "말하기 속도(1.0 보통, 기본 1.5, 저장하면 바로 적용)"),
|
||||
// The LLM (admin.spec.md, "LLM"); `llm.url` (a server the app does not start) is file-only.
|
||||
f("llm.context", Model, Int { min: 4096, max: 262144 }, json!(32768), "대화 모델이 한 번에 기억하는 양(토큰)"),
|
||||
f("brain.location", Model, Text { max: 100, non_empty: false }, json!(crate::settings::DEFAULT_LOCATION),
|
||||
f("llm.context", Llm, Int { min: 4096, max: 262144 }, json!(32768), "대화 모델이 한 번에 기억하는 양(토큰)"),
|
||||
f("brain.location", Features, Text { max: 100, non_empty: false }, json!(crate::settings::DEFAULT_LOCATION),
|
||||
"기본 위치(날씨, 검색, 추천의 기준, 비우면 자동 감지)"),
|
||||
f("llm.port", Model, Int { min: 1024, max: 65535 }, json!(8080), "대화 모델 포트"),
|
||||
f("llm.port", Llm, Int { min: 1024, max: 65535 }, json!(8080), "대화 모델 포트"),
|
||||
// The brain's features (admin.spec.md, "Features and MCP servers").
|
||||
f("features.weather", Features, Bool, json!(true), "날씨"),
|
||||
f("features.webSearch", Features, Bool, json!(true), "웹 검색"),
|
||||
@@ -141,6 +146,7 @@ pub fn fields() -> Vec<Field> {
|
||||
f("features.schedule", Features, Bool, json!(true), "예약 알림과 정기 작업"),
|
||||
f("features.notes", Features, Bool, json!(true), "사람별 지침"),
|
||||
f("features.screen", Features, Bool, json!(true), "화면 보기(Windows, PC 제어에 등록한 사람만)"),
|
||||
f("features.pcControl", Features, Bool, json!(true), "PC 제어(Windows, PC 제어에 등록한 사람만, 다음 시작부터)"),
|
||||
f("mcpServers", Features, McpServers, json!([]), "MCP 서버"),
|
||||
// Background collection (admin.spec.md, "Background collection"): no targets by default.
|
||||
f("collect.targets", Collect, CollectTargets, json!([]), "수집 대상"),
|
||||
@@ -157,7 +163,11 @@ pub fn fields() -> Vec<Field> {
|
||||
f("pcControl.type", PcControl, Choice { values: LEVELS }, json!("ask"), "글자·키 입력"),
|
||||
f("pcControl.run", PcControl, Choice { values: LEVELS }, json!("ask"), "프로그램 실행"),
|
||||
f("pcControl.screenExclude", PcControl, TextList { max_items: 100, nullable: false },
|
||||
json!(["1Password", "Bitwarden", "KeePass", "KeePassXC"]), "화면 보기에서 뺄 프로그램(파일 이름, 은행·비밀번호 관리자 등)"),
|
||||
json!(["1Password", "Bitwarden", "KeePass", "KeePassXC"]), "화면 보기와 PC 제어에서 뺄 프로그램(파일 이름, 은행·비밀번호 관리자 등)"),
|
||||
f("pcControl.antiCheatGames", PcControl, TextList { max_items: 100, nullable: false }, json!([]),
|
||||
"안티치트가 있는 게임(파일 이름, 실행 중이면 키보드·마우스 제어를 하지 않음)"),
|
||||
f("pcControl.stopHotkey", PcControl, Text { max: 40, non_empty: true }, json!(crate::pc::DEFAULT_STOP_HOTKEY),
|
||||
"비상 정지 단축키(예: Ctrl+Shift+F12, 다음 시작부터)"),
|
||||
// Game detection (admin.spec.md, "Game detection").
|
||||
f("game.detect", Game, Bool, json!(true), "게임 감지"),
|
||||
f("game.games", Game, TextList { max_items: 100, nullable: false }, json!([]), "게임으로 볼 프로그램(파일 이름, 예: MyGame.exe)"),
|
||||
|
||||
@@ -236,17 +236,17 @@ async fn warm_up(base: &str) -> Result<(), String> {
|
||||
}
|
||||
|
||||
/// The brain's settings (`config/brain.json`, the worker's `JARVIS_CONFIG_PATH`): the LLM, the
|
||||
/// memory database inside the data folder, the tools of the features that are off and the enabled
|
||||
/// MCP servers (without their environment values, `features.rs`). The app owns this file and writes
|
||||
/// it on every start.
|
||||
pub fn write_brain_config(data: &Path, base_url: &str, model: &str, settings: &crate::settings::Settings)
|
||||
/// memory database inside the data folder, the tools of the features that are off (`pc`: whether PC
|
||||
/// control is available) and the enabled MCP servers (without their environment values,
|
||||
/// `features.rs`). The app owns this file and writes it on every start.
|
||||
pub fn write_brain_config(data: &Path, base_url: &str, model: &str, settings: &crate::settings::Settings, pc: bool)
|
||||
-> Result<std::path::PathBuf, String> {
|
||||
let file = data.join("config").join("brain.json");
|
||||
let cfg = json!({
|
||||
"ollama_base_url": base_url,
|
||||
"ollama_chat_model": model,
|
||||
"db_path": data.join("memory").join("jarvis.db").to_string_lossy(),
|
||||
"disabled_tools": crate::features::disabled_tools(settings),
|
||||
"disabled_tools": crate::features::disabled_tools(settings, pc),
|
||||
"mcps": crate::features::brain_mcps(&settings.mcp_servers),
|
||||
"location_name": settings.location,
|
||||
"_config_version": 1,
|
||||
@@ -722,16 +722,17 @@ pub(crate) mod tests {
|
||||
let l = layout("llm2");
|
||||
let s = parse_settings(&json!({"features": {"weather": false},
|
||||
"mcpServers": [{"name": "brave", "command": "npx", "env": ["BRAVE_API_KEY"]}]})).unwrap();
|
||||
let f = write_brain_config(&l.data, "http://127.0.0.1:8080/v1", "m", &s).unwrap();
|
||||
let f = write_brain_config(&l.data, "http://127.0.0.1:8080/v1", "m", &s, false).unwrap();
|
||||
let v: Value = serde_json::from_str(&std::fs::read_to_string(f).unwrap()).unwrap();
|
||||
assert_eq!(v["ollama_base_url"], "http://127.0.0.1:8080/v1");
|
||||
assert_eq!(v["ollama_chat_model"], "m");
|
||||
assert!(Path::new(v["db_path"].as_str().unwrap()).starts_with(&l.data));
|
||||
assert_eq!(v["disabled_tools"], json!(crate::features::disabled_tools(&s)));
|
||||
assert_eq!(v["disabled_tools"], json!(crate::features::disabled_tools(&s, false)));
|
||||
assert!(v["disabled_tools"].as_array().unwrap().contains(&json!("pcControl")));
|
||||
assert_eq!(v["mcps"], crate::features::brain_mcps(&s.mcp_servers));
|
||||
assert_eq!(v["location_name"], crate::settings::DEFAULT_LOCATION, "the people are in Korea by default");
|
||||
let none = parse_settings(&json!({"brain": {"location": ""}})).unwrap();
|
||||
let f = write_brain_config(&l.data, "http://127.0.0.1:8080/v1", "m", &none).unwrap();
|
||||
let f = write_brain_config(&l.data, "http://127.0.0.1:8080/v1", "m", &none, false).unwrap();
|
||||
let v: Value = serde_json::from_str(&std::fs::read_to_string(f).unwrap()).unwrap();
|
||||
assert_eq!(v["location_name"], Value::Null);
|
||||
}
|
||||
|
||||
@@ -172,16 +172,54 @@ pub struct ScreenAction {
|
||||
/// `capture` or `list`.
|
||||
pub action: String,
|
||||
pub user: String,
|
||||
/// capture: `screen`, `monitor` or `window`.
|
||||
/// `voice` or `chat`.
|
||||
#[serde(default)]
|
||||
pub via: Option<String>,
|
||||
/// capture: `screen`, `monitor`, `window` or `share`.
|
||||
#[serde(default)]
|
||||
pub target: Option<String>,
|
||||
#[serde(default)]
|
||||
pub monitor: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub window: Option<String>,
|
||||
/// share: the name of the person whose screen share is asked about.
|
||||
#[serde(default)]
|
||||
pub person: Option<String>,
|
||||
/// `fast`, `normal` or `detailed`.
|
||||
#[serde(default)]
|
||||
pub detail: Option<String>,
|
||||
/// share: the name of Jarvis's voice channel, set by the core (the popped-out window's title).
|
||||
#[serde(skip)]
|
||||
pub voice_channel: Option<String>,
|
||||
}
|
||||
|
||||
/// The brain's pcControl tool asks the app for the person asking (voice_worker.spec.md, "PC control").
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq)]
|
||||
pub struct PcAction {
|
||||
pub id: String,
|
||||
/// `windows`, `read`, `click`, `type`, `keys`, `run` or `close`.
|
||||
pub action: String,
|
||||
pub user: String,
|
||||
/// `voice` or `chat`: where they asked.
|
||||
#[serde(default)]
|
||||
pub via: Option<String>,
|
||||
#[serde(default)]
|
||||
pub channel: Option<String>,
|
||||
/// A window id from `windows`, or part of a program name or title.
|
||||
#[serde(default)]
|
||||
pub window: Option<String>,
|
||||
/// An element from the latest reading (`@<snapshot>:e<n>`).
|
||||
#[serde(default, rename = "ref")]
|
||||
pub element: Option<String>,
|
||||
#[serde(default)]
|
||||
pub text: Option<String>,
|
||||
#[serde(default)]
|
||||
pub keys: Option<String>,
|
||||
#[serde(default)]
|
||||
pub program: Option<String>,
|
||||
/// The person agreed in their next message (the tool checked it).
|
||||
#[serde(default)]
|
||||
pub consent: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
@@ -197,6 +235,7 @@ pub enum WorkerEvent {
|
||||
ScheduleAction(ScheduleAction),
|
||||
NoteAction(NoteAction),
|
||||
ScreenAction(ScreenAction),
|
||||
PcAction(PcAction),
|
||||
/// The brain's answer to `reply` (empty and `failed` when it gave none).
|
||||
ReplyText { id: String, text: String, failed: bool },
|
||||
/// Was the utterance said to Jarvis? (a failed judgement is a no)
|
||||
@@ -404,6 +443,14 @@ impl WorkerClient {
|
||||
self.app_action_result("screen_action_result", id, result);
|
||||
}
|
||||
|
||||
/// The outcome of a `pc_action`: `{ok, item}`, `{ok: false, error}` or `{ok: false, ask, kinds,
|
||||
/// window}`, as `pc.rs` made it.
|
||||
pub fn pc_action_result(&self, id: &str, mut reply: Value) {
|
||||
reply["type"] = json!("pc_action_result");
|
||||
reply["id"] = json!(id);
|
||||
self.send_json(reply);
|
||||
}
|
||||
|
||||
fn app_action_result(&self, kind: &str, id: &str, result: Result<Value, String>) {
|
||||
let msg = match result {
|
||||
Ok(Value::Object(mut m)) => {
|
||||
@@ -718,6 +765,7 @@ fn on_text(inner: &Inner, t: &str) {
|
||||
"schedule_action" => serde_json::from_value(v).ok().map(WorkerEvent::ScheduleAction),
|
||||
"note_action" => serde_json::from_value(v).ok().map(WorkerEvent::NoteAction),
|
||||
"screen_action" => serde_json::from_value(v).ok().map(WorkerEvent::ScreenAction),
|
||||
"pc_action" => serde_json::from_value(v).ok().map(WorkerEvent::PcAction),
|
||||
"llm_idle" => {
|
||||
if let Some(id) = v["id"].as_u64() {
|
||||
inner.llm_idle.send_if_modified(|got| if id > *got { *got = id; true } else { false });
|
||||
|
||||
@@ -55,6 +55,15 @@ async fn the_bot_browser_reaches_public_pages_never_this_pc_and_stops() {
|
||||
assert!(!got.iter().any(|(_, t)| t.contains("secret local")));
|
||||
assert_eq!(*hits.lock().unwrap(), 0, "nothing reached this PC");
|
||||
|
||||
// Pages are not told a program drives the browser: with navigator.webdriver YouTube stopped
|
||||
// every video about 45 s in ("문제가 발생했습니다", ump.spsrejectfailure; live 2026-11-18).
|
||||
let list: serde_json::Value = serde_json::from_str(&client.get(format!("http://127.0.0.1:{}/json/list", b.cdp_port))
|
||||
.send().await.unwrap().text().await.unwrap()).unwrap();
|
||||
let page = list.as_array().unwrap().iter().find(|t| t["title"] == "Example Domain").unwrap();
|
||||
let (mut s, _) = tokio_tungstenite::connect_async(page["webSocketDebuggerUrl"].as_str().unwrap()).await.unwrap();
|
||||
let wd = cdp(&mut s, 1, "Runtime.evaluate", serde_json::json!({"expression": "navigator.webdriver"})).await;
|
||||
assert_eq!(wd["result"]["value"], false, "{wd}");
|
||||
|
||||
let port = b.cdp_port;
|
||||
let t0 = std::time::Instant::now();
|
||||
b.stop().await;
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
//! cargo test --release --test live_round_trip -- --nocapture
|
||||
//!
|
||||
//! On Linux, put the pip CUDA libraries on LD_LIBRARY_PATH (nvidia/cublas/lib, nvidia/cudnn/lib).
|
||||
//! JARVIS_LIVE_WORKER_ARGS adds worker arguments (e.g. "--stt-device cpu --stt-compute int8").
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -165,16 +166,23 @@ fn background_voices(url: String, token: String, samples: Vec<Vec<i16>>, stop: A
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn conversation_round_trip_on_real_discord() {
|
||||
if std::env::var("JARVIS_LIVE_ROUND_TRIP").as_deref() != Ok("1") {
|
||||
eprintln!("skipped: set JARVIS_LIVE_ROUND_TRIP=1");
|
||||
return;
|
||||
}
|
||||
/// Both bots in the voice channel: the speaker bot (the "person") and Jarvis running the app's services
|
||||
/// from a fresh data folder, with `call` as the settings' call section.
|
||||
struct Live {
|
||||
speaker_id: u64,
|
||||
speaker_songbird: Arc<Songbird>,
|
||||
s_shards: Arc<serenity::gateway::ShardManager>,
|
||||
services: jarvis_app::services::Services,
|
||||
core: jarvis_app::discord::DiscordCore,
|
||||
events: Arc<Mutex<Vec<(Instant, CoreEvent)>>>,
|
||||
call: Arc<tokio::sync::Mutex<songbird::Call>>,
|
||||
ears: Ears,
|
||||
guild: u64,
|
||||
}
|
||||
|
||||
async fn start_live(call_settings: impl FnOnce(u64) -> Value) -> Live {
|
||||
let guild: u64 = env("JARVIS_LIVE_GUILD").parse().unwrap();
|
||||
let voice: u64 = env("JARVIS_LIVE_VOICE_CHANNEL").parse().unwrap();
|
||||
let rounds: usize = std::env::var("JARVIS_LIVE_ROUNDS").ok().and_then(|v| v.parse().ok()).unwrap_or(2);
|
||||
let questions = PathBuf::from(env("JARVIS_LIVE_QUESTIONS"));
|
||||
env("JARVIS_LIVE_LLM_URL"); // the services point the brain at it (llm.url)
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
|
||||
@@ -208,21 +216,26 @@ async fn conversation_round_trip_on_real_discord() {
|
||||
if let Ok(m) = std::env::var("JARVIS_MELO_PYTHON") {
|
||||
args.extend(["--melo-python".to_string(), m]);
|
||||
}
|
||||
// e.g. "--stt-device cpu --stt-compute int8 --melo-device cpu" on a host without a GPU
|
||||
if let Ok(extra) = std::env::var("JARVIS_LIVE_WORKER_ARGS") {
|
||||
args.extend(extra.split_whitespace().map(String::from));
|
||||
}
|
||||
std::fs::write(data.join("config").join("settings.json"), serde_json::to_string_pretty(&json!({
|
||||
"discord": {"guildId": guild.to_string(), "voiceChannelId": voice.to_string()},
|
||||
// The "person" here is a bot, so bots must be heard.
|
||||
"voice": {"listenToBots": true},
|
||||
// The "person" here is a bot, so bots must be heard; the recorded questions call "자비스".
|
||||
"voice": {"listenToBots": true, "wakeWords": ["자비스"]},
|
||||
"call": call_settings(speaker_id),
|
||||
"voiceWorker": {"python": env("JARVIS_VOICE_PYTHON"), "cwd": root.join("../src"), "args": args},
|
||||
"llm": {"url": env("JARVIS_LIVE_LLM_URL")},
|
||||
})).unwrap()).unwrap();
|
||||
std::env::set_var("JARVIS_DISCORD_TOKEN", &bot_token);
|
||||
let services = tokio::time::timeout(Duration::from_secs(300), start_services(&data, Arc::new(Llm::new(&data)))).await
|
||||
let mut services = tokio::time::timeout(Duration::from_secs(300), start_services(&data, Arc::new(Llm::new(&data)))).await
|
||||
.expect("services start in time").expect("services start");
|
||||
services.worker.wait_ready(Duration::from_secs(240)).await.expect("worker ready");
|
||||
let core = services.core.clone();
|
||||
let events: Arc<Mutex<Vec<(Instant, CoreEvent)>>> = Arc::default();
|
||||
let ev2 = events.clone();
|
||||
let mut rx = services.events;
|
||||
let mut rx = std::mem::replace(&mut services.events, mpsc::unbounded_channel().1);
|
||||
tokio::spawn(async move {
|
||||
while let Some(e) = rx.recv().await {
|
||||
ev2.lock().unwrap().push((Instant::now(), e));
|
||||
@@ -240,6 +253,31 @@ async fn conversation_round_trip_on_real_discord() {
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(2)).await; // the DAVE group settles after the join
|
||||
|
||||
Live { speaker_id, speaker_songbird, s_shards, services, core, events, call, ears, guild }
|
||||
}
|
||||
|
||||
impl Live {
|
||||
async fn stop(self) {
|
||||
let _ = self.speaker_songbird.remove(GuildId::new(self.guild)).await;
|
||||
self.s_shards.shutdown_all().await;
|
||||
self.services.core.leave("test").await;
|
||||
self.services.core.close().await;
|
||||
self.services.worker.close();
|
||||
self.services.worker_process.stop().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn conversation_round_trip_on_real_discord() {
|
||||
if std::env::var("JARVIS_LIVE_ROUND_TRIP").as_deref() != Ok("1") {
|
||||
eprintln!("skipped: set JARVIS_LIVE_ROUND_TRIP=1");
|
||||
return;
|
||||
}
|
||||
let live = start_live(|_| json!({})).await;
|
||||
let rounds: usize = std::env::var("JARVIS_LIVE_ROUNDS").ok().and_then(|v| v.parse().ok()).unwrap_or(2);
|
||||
let questions = PathBuf::from(env("JARVIS_LIVE_QUESTIONS"));
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let Live { speaker_id, ref services, ref core, ref events, ref call, ref ears, .. } = live;
|
||||
let refs: Value = serde_json::from_str(&std::fs::read_to_string(questions.join("refs.json")).unwrap()).unwrap();
|
||||
let qs: Vec<(String, Vec<u8>)> = refs["samples"].as_array().unwrap().iter()
|
||||
.map(|r| (r["text"].as_str().unwrap().to_owned(), std::fs::read(questions.join(r["file"].as_str().unwrap())).unwrap()))
|
||||
@@ -355,12 +393,7 @@ async fn conversation_round_trip_on_real_discord() {
|
||||
println!("live round trip {}", serde_json::to_string_pretty(&Value::Object(report.clone())).unwrap());
|
||||
})).await;
|
||||
|
||||
let _ = speaker_songbird.remove(GuildId::new(guild)).await;
|
||||
s_shards.shutdown_all().await;
|
||||
services.core.leave("test").await;
|
||||
services.core.close().await;
|
||||
services.worker.close();
|
||||
services.worker_process.stop().await;
|
||||
live.stop().await;
|
||||
if let Err(panic) = steps {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
@@ -371,3 +404,76 @@ async fn conversation_round_trip_on_real_discord() {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// Going on (conversation.spec.md), live: the person's own recordings of one thing said in pieces,
|
||||
/// played with the pauses they really made, get one answer. Settings as the person's own app: they
|
||||
/// are the only designated person, `addressed` mode, alone with Jarvis.
|
||||
///
|
||||
/// JARVIS_LIVE_GOING_ON=1 (the env of conversation_round_trip_on_real_discord, without the questions)
|
||||
/// JARVIS_LIVE_PIECES=<dir with pieces.json: [{"name", "file"}], each file one 16 kHz mono wav of
|
||||
/// the pieces and the silence between them> cargo test --release --test live_round_trip going_on -- --nocapture
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn going_on_on_real_discord() {
|
||||
if std::env::var("JARVIS_LIVE_GOING_ON").as_deref() != Ok("1") {
|
||||
eprintln!("skipped: set JARVIS_LIVE_GOING_ON=1");
|
||||
return;
|
||||
}
|
||||
let dir = PathBuf::from(env("JARVIS_LIVE_PIECES"));
|
||||
let seqs: Value = serde_json::from_str(&std::fs::read_to_string(dir.join("pieces.json")).unwrap()).unwrap();
|
||||
let live = start_live(|speaker| json!({"designated": [speaker.to_string()], "designatedOnly": true, "mode": "addressed"})).await;
|
||||
let mut rows = vec![];
|
||||
let steps = futures_util::FutureExt::catch_unwind(std::panic::AssertUnwindSafe(async {
|
||||
for s in seqs.as_array().unwrap() {
|
||||
let wav = std::fs::read(dir.join(s["file"].as_str().unwrap())).unwrap();
|
||||
until("Jarvis to finish speaking", 60_000, || !live.core.speaking()).await;
|
||||
let before = live.events.lock().unwrap().len();
|
||||
live.call.lock().await.play_only_input(Input::from(wav.clone()));
|
||||
tokio::time::sleep(Duration::from_secs_f64(speech_end_seconds(&wav))).await;
|
||||
// Done when nothing is playing and nothing new came for 10 s (at most 60 s).
|
||||
let deadline = Instant::now() + Duration::from_secs(60);
|
||||
let (mut seen, mut quiet_since) = (0, Instant::now());
|
||||
while Instant::now() < deadline {
|
||||
let n = live.events.lock().unwrap().len();
|
||||
if n != seen || live.core.speaking() {
|
||||
(seen, quiet_since) = (n, Instant::now());
|
||||
} else if quiet_since.elapsed() > Duration::from_secs(10) {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
let ev = live.events.lock().unwrap()[before..].to_vec();
|
||||
let heard: Vec<String> = ev.iter().filter_map(|(_, e)| match e {
|
||||
CoreEvent::Final(f) if f.heard.speaker_id == live.speaker_id => Some(f.text.clone()),
|
||||
_ => None,
|
||||
}).collect();
|
||||
let answers: Vec<Value> = ev.iter().filter_map(|(_, e)| match e {
|
||||
CoreEvent::ReplyText { id, text, failed } => {
|
||||
let end = ev.iter().find_map(|(_, e)| match e {
|
||||
CoreEvent::SpeakEnd(x) if &x.id == id => Some(x.clone()),
|
||||
_ => None,
|
||||
});
|
||||
Some(json!({"text": text, "failed": failed, "played": end.as_ref().map(|x| x.played),
|
||||
"cancelled": end.as_ref().map(|x| x.cancelled)}))
|
||||
}
|
||||
_ => None,
|
||||
}).collect();
|
||||
let asked = ev.iter().filter(|(_, e)| matches!(e, CoreEvent::AnswerAsked { .. })).count();
|
||||
rows.push(json!({"name": s["name"], "heard": heard, "asked": asked, "answers": answers}));
|
||||
println!("going on {}", rows.last().unwrap());
|
||||
tokio::time::sleep(Duration::from_millis(1500)).await;
|
||||
}
|
||||
})).await;
|
||||
live.stop().await;
|
||||
if let Err(panic) = steps {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
// One thing said in pieces: one answer heard to its end, and no other answer heard.
|
||||
for r in &rows {
|
||||
let heard = |a: &&Value| a["played"].as_u64().is_some_and(|p| p > 0);
|
||||
let whole: Vec<&Value> = r["answers"].as_array().unwrap().iter().filter(heard)
|
||||
.filter(|a| a["cancelled"] == json!(false)).collect();
|
||||
let cut: Vec<&Value> = r["answers"].as_array().unwrap().iter().filter(heard)
|
||||
.filter(|a| a["cancelled"] == json!(true)).collect();
|
||||
assert!(whole.len() == 1 && cut.is_empty(), "{}: not one answer: {r}", r["name"]);
|
||||
}
|
||||
}
|
||||
|
||||
160
app/tests/pc_keys_windows.rs
Normal file
160
app/tests/pc_keys_windows.rs
Normal file
@@ -0,0 +1,160 @@
|
||||
//! PC control's keys on a real Windows desktop (admin.spec.md, "PC control", "Keys"): with another
|
||||
//! window in front, letters and Enter reach Notepad behind it and the window in front stays; a
|
||||
//! combination and the Calculator's keys go in front for a moment and the window in front comes
|
||||
//! back. Runs only on Windows with `JARVIS_TEST_PC_KEYS=1` and agent-desktop at
|
||||
//! `JARVIS_AGENT_DESKTOP` (it opens Notepad, Character Map and the Calculator); elsewhere it skips.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use jarvis_app::pc::{Cli, Desktop, Pc, Stop};
|
||||
use jarvis_app::settings::{Level, Settings};
|
||||
use jarvis_app::worker_client::PcAction;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
const USER: &str = "42";
|
||||
|
||||
fn act(pc: &Pc, action: &str, extra: Value) -> Value {
|
||||
let mut v = json!({"id": "k1", "action": action, "user": USER, "via": "voice"});
|
||||
v.as_object_mut().unwrap().extend(extra.as_object().unwrap().clone());
|
||||
let a: PcAction = serde_json::from_value(v).unwrap();
|
||||
let t = Instant::now();
|
||||
let o = pc.act(&a);
|
||||
println!("{action} {extra}: {} in {} ms", o.code, t.elapsed().as_millis());
|
||||
o.reply
|
||||
}
|
||||
|
||||
/// Every element's name and value of a reading, joined.
|
||||
fn text(reply: &Value) -> String {
|
||||
reply["item"]["elements"].as_array().into_iter().flatten()
|
||||
.map(|e| format!("{} {}", e["name"].as_str().unwrap_or(""), e["value"].as_str().unwrap_or(""))).collect::<Vec<_>>().join(" | ")
|
||||
}
|
||||
|
||||
/// Every element's value of a reading, joined: the typed text without the window's labels (a label
|
||||
/// such as "Ctrl+Shift+X" contains "hi").
|
||||
fn values(reply: &Value) -> String {
|
||||
reply["item"]["elements"].as_array().into_iter().flatten()
|
||||
.filter_map(|e| e["value"].as_str()).collect::<Vec<_>>().join(" | ")
|
||||
}
|
||||
|
||||
/// Closed however the test ends (by name too: a program may hand over to another process).
|
||||
struct Closed(Vec<String>, Vec<std::process::Child>);
|
||||
|
||||
impl Drop for Closed {
|
||||
fn drop(&mut self) {
|
||||
for p in &self.0 {
|
||||
let _ = std::process::Command::new("taskkill").args(["/f", "/im", p]).output();
|
||||
}
|
||||
for c in &mut self.1 {
|
||||
let _ = c.kill();
|
||||
let _ = c.wait();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimise window `id` ("w-<handle>") as the person would.
|
||||
#[cfg(windows)]
|
||||
fn minimise(id: &str) {
|
||||
let h: usize = id.trim_start_matches("w-").parse().unwrap();
|
||||
// SAFETY: a plain window call on a handle from list-windows.
|
||||
unsafe { windows_sys::Win32::UI::WindowsAndMessaging::ShowWindow(h as _, windows_sys::Win32::UI::WindowsAndMessaging::SW_MINIMIZE) };
|
||||
}
|
||||
#[cfg(not(windows))]
|
||||
fn minimise(_id: &str) {}
|
||||
|
||||
fn wait_window(pc: &Pc, f: impl Fn(&Value) -> bool) -> String {
|
||||
let until = Instant::now() + Duration::from_secs(15);
|
||||
loop {
|
||||
let list = act(pc, "windows", json!({}));
|
||||
if let Some(w) = list["item"]["windows"].as_array().into_iter().flatten().find(|w| f(w)) {
|
||||
return w["id"].as_str().unwrap().to_owned();
|
||||
}
|
||||
assert!(Instant::now() < until, "the window did not show");
|
||||
std::thread::sleep(Duration::from_millis(500));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keys_go_behind_or_in_front_for_a_moment_and_the_window_in_front_stays() {
|
||||
if std::env::var("JARVIS_TEST_PC_KEYS").is_err() {
|
||||
eprintln!("skipped: set JARVIS_TEST_PC_KEYS=1 and JARVIS_AGENT_DESKTOP");
|
||||
return;
|
||||
}
|
||||
let exe = PathBuf::from(std::env::var("JARVIS_AGENT_DESKTOP").expect("JARVIS_AGENT_DESKTOP"));
|
||||
let data = std::env::temp_dir().join(format!("jarvis-pc-keys-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&data).unwrap();
|
||||
let mut s = Settings::default();
|
||||
s.pc_control.allowed_users = vec![USER.parse().unwrap()];
|
||||
s.pc_control.type_text = Level::Auto;
|
||||
s.pc_control.run = Level::Auto;
|
||||
let stop: &'static Stop = Box::leak(Box::new(Stop::new()));
|
||||
let desktop = Cli::new(&exe, &data);
|
||||
let probe = Cli::new(&exe, &data.join("probe"));
|
||||
let pc = Pc::new(s, Box::new(desktop), stop, Box::new(|| None));
|
||||
// Only programs this test starts are closed: none of them may run before it.
|
||||
let ours = ["notepad.exe", "charmap.exe", "calculatorapp.exe", "win32calc.exe"];
|
||||
let running: Vec<String> = probe.processes().into_iter().filter(|p| ours.contains(&p.as_str())).collect();
|
||||
assert!(running.is_empty(), "close these first: {running:?}");
|
||||
let mut closed = Closed(ours.iter().map(|p| p.to_string()).collect(), vec![]);
|
||||
|
||||
let file = data.join("pckeys.txt");
|
||||
std::fs::write(&file, "").unwrap();
|
||||
closed.1.push(std::process::Command::new("notepad.exe").arg(&file).spawn().unwrap());
|
||||
let notepad = wait_window(&pc, |w| w["title"].as_str().is_some_and(|t| t.contains("pckeys")));
|
||||
closed.1.push(std::process::Command::new("charmap.exe").spawn().unwrap());
|
||||
let front = wait_window(&pc, |w| w["program"] == "charmap");
|
||||
std::thread::sleep(Duration::from_secs(1));
|
||||
let _ = probe.run(&["focus-window".into(), "--window-id".into(), front.clone()], None, stop);
|
||||
println!("notepad {notepad}; in front {front}: {:?}", probe.foreground());
|
||||
assert_eq!(probe.foreground().as_deref(), Some(front.as_str()), "Character Map is in front");
|
||||
assert!(probe.takes_keys_behind(¬epad), "Notepad takes keys behind");
|
||||
|
||||
// Letters and Enter behind Notepad: they arrive and the window in front stays.
|
||||
for k in ["h", "i", "enter", "o", "k"] {
|
||||
let r = act(&pc, "keys", json!({"window": notepad, "keys": k}));
|
||||
assert_eq!(r["ok"], true, "{k}: {r}");
|
||||
assert_eq!(probe.foreground().as_deref(), Some(front.as_str()), "{k}: the window in front stays");
|
||||
}
|
||||
let read = act(&pc, "read", json!({"window": notepad}));
|
||||
println!("notepad after behind: {}", text(&read).chars().take(300).collect::<String>());
|
||||
assert!(values(&read).contains("hi") && values(&read).contains("ok"), "the letters reached Notepad behind");
|
||||
|
||||
// A combination goes in front for a moment, then the window in front is back.
|
||||
let r = act(&pc, "keys", json!({"window": notepad, "keys": "ctrl+a"}));
|
||||
assert_eq!(r["ok"], true, "ctrl+a: {r}");
|
||||
assert_eq!(probe.foreground().as_deref(), Some(front.as_str()), "ctrl+a: the window in front is back");
|
||||
let r = act(&pc, "keys", json!({"window": notepad, "keys": "x"}));
|
||||
assert_eq!(r["ok"], true, "x: {r}");
|
||||
let after = values(&act(&pc, "read", json!({"window": notepad})));
|
||||
println!("notepad after ctrl+a, x: {after:?}");
|
||||
assert!(after.split(" | ").any(|v| v == "x") && !after.contains("hi"), "ctrl+a selected the text and x replaced it");
|
||||
|
||||
// A minimised window: restored in the moment in front, and the window in front is back.
|
||||
minimise(¬epad);
|
||||
std::thread::sleep(Duration::from_millis(500));
|
||||
let _ = probe.run(&["focus-window".into(), "--window-id".into(), front.clone()], None, stop);
|
||||
let r = act(&pc, "keys", json!({"window": notepad, "keys": "ctrl+a"}));
|
||||
assert_eq!(r["ok"], true, "minimised ctrl+a: {r}");
|
||||
assert_eq!(probe.foreground().as_deref(), Some(front.as_str()), "minimised ctrl+a: the window in front is back");
|
||||
let r = act(&pc, "keys", json!({"window": notepad, "keys": "y"}));
|
||||
assert_eq!(r["ok"], true, "y: {r}");
|
||||
let after = values(&act(&pc, "read", json!({"window": notepad})));
|
||||
println!("notepad after minimised, ctrl+a, y: {after:?}");
|
||||
assert!(after.split(" | ").any(|v| v == "y") && !after.split(" | ").any(|v| v == "x"), "ctrl+a reached the restored Notepad");
|
||||
|
||||
// The Calculator: in front for a moment when it does not take keys behind (the Store app).
|
||||
let calc = act(&pc, "run", json!({"program": "calc"}));
|
||||
let calc = calc["item"]["window"]["id"].as_str().expect("the Calculator's window").to_owned();
|
||||
std::thread::sleep(Duration::from_secs(1));
|
||||
let _ = probe.run(&["focus-window".into(), "--window-id".into(), front.clone()], None, stop);
|
||||
println!("calculator {calc}; takes keys behind: {}", probe.takes_keys_behind(&calc));
|
||||
for k in ["escape", "1", "2"] {
|
||||
let r = act(&pc, "keys", json!({"window": calc, "keys": k}));
|
||||
assert_eq!(r["ok"], true, "{k}: {r}");
|
||||
assert_eq!(probe.foreground().as_deref(), Some(front.as_str()), "{k}: the window in front stays or is back");
|
||||
}
|
||||
let shown = text(&act(&pc, "read", json!({"window": calc})));
|
||||
println!("calculator after 1 2: {}", shown.chars().take(300).collect::<String>());
|
||||
assert!(shown.contains("12"), "the Calculator shows 12");
|
||||
let _ = std::fs::remove_dir_all(&data);
|
||||
}
|
||||
@@ -340,6 +340,26 @@ async function scheduleList() {
|
||||
h("div", { class: "field" }, h("button", { class: "btn", onclick: load }, "새로 고침")), list);
|
||||
}
|
||||
|
||||
/** PC control's emergency stop (admin.spec.md, "PC control"): its state and 풀기. */
|
||||
async function pcStopCard() {
|
||||
const state = h("div");
|
||||
const load = async () => {
|
||||
try {
|
||||
const { stopped } = await api("pc_stop_status");
|
||||
state.replaceChildren(stopped
|
||||
? h("div", {}, h("span", { class: "badge bad" }, "비상 정지 중"), " ", h("button", { class: "btn", onclick: async () => {
|
||||
try { await api("pc_stop_lift"); toast("비상 정지를 풀었습니다"); load(); } catch (e) { fail(e); }
|
||||
} }, "풀기"))
|
||||
: h("div", { class: "muted" }, "비상 정지 중이 아닙니다"));
|
||||
} catch (e) { fail(e); }
|
||||
};
|
||||
load();
|
||||
return h("div", { class: "card" }, h("h2", {}, "비상 정지"),
|
||||
h("div", { class: "hint" }, "비상 정지 단축키를 누르면 하던 조작을 바로 멈추고, 다시 누르거나 여기서 풀 때까지 화면 보기와 창 목록 말고는 하지 않습니다. " +
|
||||
"앱을 다시 시작해도 풀립니다. 안티치트가 있는 게임이 실행 중일 때도 키보드·마우스 제어는 하지 않습니다."),
|
||||
h("div", { class: "field" }, h("button", { class: "btn", onclick: load }, "새로 고침")), state);
|
||||
}
|
||||
|
||||
/** What is never done, and the chat channels' send and delete switches, beside the PC control levels. */
|
||||
async function permissionsCard() {
|
||||
const view = await api("settings_view");
|
||||
@@ -802,7 +822,8 @@ function dashboardBody(d) {
|
||||
h("span", { class: "k" }, "게임 모드"), h("span", { class: d.game ? "" : "muted" },
|
||||
d.game ? `켜짐 (${d.game.name}, ${GAME_REASON[d.game.reason.kind]?.(d.game.reason) ?? d.game.reason.kind})` : "꺼짐"),
|
||||
h("span", { class: "k" }, "성능 단계"), h("span", { class: d.performance && d.performance.stage !== "normal" ? "" : "muted" },
|
||||
perfText(d.performance))))),
|
||||
perfText(d.performance)),
|
||||
d.pcStop ? [h("span", { class: "k" }, "PC 제어"), h("span", {}, h("span", { class: "badge bad" }, "비상 정지"))] : null))),
|
||||
h("div", { class: "grid" },
|
||||
h("div", { class: "card" }, h("h2", {}, "실시간 자막"), h("div", { class: "captions" },
|
||||
(d.captions || []).length ? d.captions.map((c) => h("div", { class: "cap" + (c.final ? "" : " partial") }, h("b", {}, c.name), c.text))
|
||||
@@ -971,7 +992,7 @@ function updateCard() {
|
||||
async function checkUpdate() {
|
||||
try {
|
||||
const u = await api("update_check");
|
||||
if (u.latest) { $("#nav-app").textContent = "앱 (새 버전)"; toast(`새 버전 ${u.latest.version}이 있습니다. 앱 화면에서 설치할 수 있습니다.`); }
|
||||
if (u.latest) { $("#nav-app").textContent = "앱 (새 버전)"; setGroup($("#nav-app").closest(".group"), true); toast(`새 버전 ${u.latest.version}이 있습니다. 앱 화면에서 설치할 수 있습니다.`); }
|
||||
} catch (e) { /* offline: the 앱 screen shows why */ }
|
||||
}
|
||||
|
||||
@@ -1004,27 +1025,48 @@ async function show(name) {
|
||||
current = name;
|
||||
clearInterval(ticker);
|
||||
for (const b of document.querySelectorAll(".nav")) b.classList.toggle("on", b.dataset.screen === name);
|
||||
// The group of the screen shown opens (a new version's 앱 button is then seen too).
|
||||
const group = document.querySelector(`.nav[data-screen="${name}"]`)?.closest(".group");
|
||||
if (group) setGroup(group, true);
|
||||
const screen = $("#screen");
|
||||
try {
|
||||
if (name === "dashboard") { screen.replaceChildren(); await refresh(); ticker = setInterval(refresh, 1000); return; }
|
||||
if (name === "logs") { const l = logsScreen(); screen.replaceChildren(l.view); ticker = setInterval(l.tick, 1000); return; }
|
||||
if (name === "conversations") { const c = conversationScreen(); screen.replaceChildren(c.view); ticker = setInterval(c.tick, 500); return; }
|
||||
if (name === "notes") { screen.replaceChildren(notesScreen()); return; }
|
||||
const titles = { bot: "봇", channels: "채널·호출", chat: "채팅 채널", memory: "대화 기억", model: "모델", features: "기능",
|
||||
collect: "백그라운드 수집", game: "게임 모드", schedule: "예약", pcControl: "PC 제어", app: "앱" };
|
||||
const titles = { bot: "봇", voiceChannel: "음성 채널", call: "호출", response: "여럿이 말할 때", chat: "채팅 채널",
|
||||
memory: "대화 기억", voice: "받아 적기·말하기", llm: "대화 모델", features: "기능 켜기·MCP",
|
||||
collect: "백그라운드 수집", game: "게임 모드", schedule: "예약", pcControl: "PC 제어·화면 보기", app: "앱" };
|
||||
const update = name === "app" ? updateCard() : null;
|
||||
const extra = name === "bot" ? await tokenCard() : name === "features" ? h("div", {}, await kmaKeyCard(), await displayDriverCard())
|
||||
: name === "app" ? h("div", {}, update.view, h("div", { class: "hint", style: "margin:12px 0" }, "트레이로 최소화하면 창만 숨고 봇과 서비스는 계속 돕니다. 작업 표시줄 오른쪽의 Jarvis 아이콘을 누르면 창이 다시 열리고, 아이콘의 오른쪽 클릭 메뉴에서 완전 종료할 수 있습니다. 저장하면 바로 적용됩니다.")) : null;
|
||||
const view = await settingsScreen(name, titles[name], extra);
|
||||
if (name === "collect") view.append(await collectResults());
|
||||
if (name === "schedule") view.append(await scheduleList());
|
||||
if (name === "pcControl") view.append(await permissionsCard());
|
||||
if (name === "pcControl") view.append(await pcStopCard(), await permissionsCard());
|
||||
if (my !== gen) return;
|
||||
screen.replaceChildren(view);
|
||||
if (update) ticker = setInterval(update.tick, 1000);
|
||||
} catch (e) { fail(e); }
|
||||
}
|
||||
|
||||
/** Open or close one menu group; which groups are open is kept for the next start. */
|
||||
function setGroup(g, open) {
|
||||
g.classList.toggle("open", open);
|
||||
g.querySelector(".group-head").setAttribute("aria-expanded", String(open));
|
||||
const kept = new Set(JSON.parse(localStorage.getItem("openGroups") || "[]"));
|
||||
open ? kept.add(g.dataset.group) : kept.delete(g.dataset.group);
|
||||
localStorage.setItem("openGroups", JSON.stringify([...kept]));
|
||||
}
|
||||
|
||||
function initGroups() {
|
||||
const kept = new Set(JSON.parse(localStorage.getItem("openGroups") || "[]"));
|
||||
for (const g of document.querySelectorAll(".group")) {
|
||||
if (kept.has(g.dataset.group)) setGroup(g, true);
|
||||
g.querySelector(".group-head").onclick = () => setGroup(g, !g.classList.contains("open"));
|
||||
}
|
||||
}
|
||||
|
||||
// ---- first-run wizard ---------------------------------------------------------------------------
|
||||
|
||||
const WIZ = ["데이터 폴더", "PC 점검", "모델", "디스코드 봇", "서버·채널", "호출어", "완료"];
|
||||
@@ -1192,6 +1234,7 @@ async function start() {
|
||||
$("#wizard").hidden = true;
|
||||
$("#main").hidden = false;
|
||||
for (const b of document.querySelectorAll(".nav")) b.onclick = () => show(b.dataset.screen);
|
||||
initGroups();
|
||||
show("dashboard");
|
||||
checkUpdate();
|
||||
}
|
||||
|
||||
@@ -13,19 +13,52 @@
|
||||
<div class="brand">Jarvis</div>
|
||||
<button data-screen="dashboard" class="nav">대시보드</button>
|
||||
<button data-screen="conversations" class="nav">대화</button>
|
||||
<button data-screen="bot" class="nav">봇</button>
|
||||
<button data-screen="channels" class="nav">채널·호출</button>
|
||||
<button data-screen="chat" class="nav">채팅 채널</button>
|
||||
<button data-screen="memory" class="nav">대화 기억</button>
|
||||
<button data-screen="model" class="nav">모델</button>
|
||||
<button data-screen="features" class="nav">기능</button>
|
||||
<button data-screen="collect" class="nav">백그라운드 수집</button>
|
||||
<button data-screen="schedule" class="nav">예약</button>
|
||||
<button data-screen="pcControl" class="nav">PC 제어</button>
|
||||
<button data-screen="notes" class="nav">사람별 지침</button>
|
||||
<button data-screen="game" class="nav">게임 모드</button>
|
||||
<button data-screen="logs" class="nav">로그</button>
|
||||
<button data-screen="app" class="nav" id="nav-app">앱</button>
|
||||
<div class="group" data-group="discord">
|
||||
<button class="group-head" aria-expanded="false">디스코드</button>
|
||||
<div class="group-items">
|
||||
<button data-screen="bot" class="nav">봇</button>
|
||||
<button data-screen="voiceChannel" class="nav">음성 채널</button>
|
||||
<button data-screen="chat" class="nav">채팅 채널</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="group" data-group="talk">
|
||||
<button class="group-head" aria-expanded="false">대화 방식</button>
|
||||
<div class="group-items">
|
||||
<button data-screen="call" class="nav">호출</button>
|
||||
<button data-screen="response" class="nav">여럿이 말할 때</button>
|
||||
<button data-screen="memory" class="nav">대화 기억</button>
|
||||
<button data-screen="notes" class="nav">사람별 지침</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="group" data-group="model">
|
||||
<button class="group-head" aria-expanded="false">음성·모델</button>
|
||||
<div class="group-items">
|
||||
<button data-screen="voice" class="nav">받아 적기·말하기</button>
|
||||
<button data-screen="llm" class="nav">대화 모델</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="group" data-group="features">
|
||||
<button class="group-head" aria-expanded="false">기능</button>
|
||||
<div class="group-items">
|
||||
<button data-screen="features" class="nav">기능 켜기·MCP</button>
|
||||
<button data-screen="schedule" class="nav">예약</button>
|
||||
<button data-screen="collect" class="nav">백그라운드 수집</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="group" data-group="pc">
|
||||
<button class="group-head" aria-expanded="false">PC</button>
|
||||
<div class="group-items">
|
||||
<button data-screen="pcControl" class="nav">PC 제어·화면 보기</button>
|
||||
<button data-screen="game" class="nav">게임 모드</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="group" data-group="admin">
|
||||
<button class="group-head" aria-expanded="false">관리</button>
|
||||
<div class="group-items">
|
||||
<button data-screen="logs" class="nav">로그</button>
|
||||
<button data-screen="app" class="nav" id="nav-app">앱</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="side-foot" id="version"></div>
|
||||
</nav>
|
||||
<main id="screen" class="screen"></main>
|
||||
|
||||
@@ -17,6 +17,14 @@ button, input, select, textarea { font: inherit; color: inherit; }
|
||||
.nav { text-align: left; background: none; border: 0; padding: 8px 10px; border-radius: 6px; color: var(--ink2); cursor: pointer; }
|
||||
.nav:hover { background: var(--s2); }
|
||||
.nav.on { background: var(--s3); color: var(--ink); }
|
||||
.group { display: flex; flex-direction: column; margin-top: 6px; }
|
||||
.group-head { text-align: left; background: none; border: 0; padding: 8px 10px; border-radius: 6px; color: var(--ink3); cursor: pointer; font-size: 12px; font-weight: 600; letter-spacing: 0.2px; display: flex; align-items: center; gap: 6px; }
|
||||
.group-head::before { content: "▸"; font-size: 10px; color: var(--ink4); }
|
||||
.group-head[aria-expanded="true"]::before { content: "▾"; }
|
||||
.group-head:hover { background: var(--s2); color: var(--ink2); }
|
||||
.group-items { display: none; flex-direction: column; gap: 2px; }
|
||||
.group.open .group-items { display: flex; }
|
||||
.group-items .nav { padding-left: 24px; }
|
||||
.side-foot { margin-top: auto; color: var(--ink4); font-size: 12px; padding: 8px 10px; }
|
||||
.screen { padding: 28px 32px; max-width: 1100px; }
|
||||
h1 { font-size: 24px; font-weight: 600; letter-spacing: -0.4px; margin: 0 0 18px; }
|
||||
|
||||
6
app/vendor/songbird/.cargo_vcs_info.json
vendored
6
app/vendor/songbird/.cargo_vcs_info.json
vendored
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"git": {
|
||||
"sha1": "957e17fcbc687cd88979c90b76bdf79017803b09"
|
||||
},
|
||||
"path_in_vcs": ""
|
||||
}
|
||||
376
app/vendor/songbird/Cargo.toml
vendored
376
app/vendor/songbird/Cargo.toml
vendored
@@ -1,376 +0,0 @@
|
||||
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
|
||||
#
|
||||
# When uploading crates to the registry Cargo will automatically
|
||||
# "normalize" Cargo.toml files for maximal compatibility
|
||||
# with all versions of Cargo and also rewrite `path` dependencies
|
||||
# to registry (e.g., crates.io) dependencies.
|
||||
#
|
||||
# If you are reading this file be aware that the original Cargo.toml
|
||||
# will likely look very different (and much more reasonable).
|
||||
# See Cargo.toml.orig for the original contents.
|
||||
|
||||
[package]
|
||||
edition = "2021"
|
||||
rust-version = "1.83.0"
|
||||
name = "songbird"
|
||||
version = "0.6.1"
|
||||
authors = ["Kyle Simpson <kyleandrew.simpson@gmail.com>"]
|
||||
build = "build.rs"
|
||||
include = [
|
||||
"src/**/*.rs",
|
||||
"Cargo.toml",
|
||||
"build.rs",
|
||||
]
|
||||
autolib = false
|
||||
autobins = false
|
||||
autoexamples = false
|
||||
autotests = false
|
||||
autobenches = false
|
||||
description = "An async Rust library for the Discord voice API."
|
||||
homepage = "https://github.com/serenity-rs/songbird"
|
||||
documentation = "https://docs.rs/songbird"
|
||||
readme = "README.md"
|
||||
keywords = [
|
||||
"discord",
|
||||
"api",
|
||||
"rtp",
|
||||
"audio",
|
||||
]
|
||||
license = "ISC"
|
||||
repository = "https://github.com/serenity-rs/songbird.git"
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
features = ["full-doc"]
|
||||
rustdoc-args = [
|
||||
"--cfg",
|
||||
"docsrs",
|
||||
]
|
||||
|
||||
[features]
|
||||
builtin-queue = []
|
||||
default = [
|
||||
"driver",
|
||||
"gateway",
|
||||
"rustls",
|
||||
"serenity",
|
||||
"tungstenite",
|
||||
]
|
||||
driver = [
|
||||
"dep:aead",
|
||||
"dep:aes-gcm",
|
||||
"dep:async-trait",
|
||||
"dep:opus2",
|
||||
"dep:byteorder",
|
||||
"dep:bytes",
|
||||
"dep:chacha20poly1305",
|
||||
"dep:crypto-common",
|
||||
"dep:davey",
|
||||
"dep:discortp",
|
||||
"dep:flume",
|
||||
"dep:nohash-hasher",
|
||||
"dep:parking_lot",
|
||||
"dep:rand",
|
||||
"dep:reqwest",
|
||||
"dep:ringbuf",
|
||||
"dep:rubato",
|
||||
"dep:rusty_pool",
|
||||
"dep:serde-aux",
|
||||
"dep:serenity-voice-model",
|
||||
"dep:socket2",
|
||||
"dep:stream_lib",
|
||||
"dep:streamcatcher",
|
||||
"dep:symphonia",
|
||||
"dep:symphonia-core",
|
||||
"dep:tokio",
|
||||
"dep:tokio-util",
|
||||
"dep:typenum",
|
||||
"dep:url",
|
||||
"dep:uuid",
|
||||
"tokio?/fs",
|
||||
"tokio?/io-util",
|
||||
"tokio?/macros",
|
||||
"tokio?/net",
|
||||
"tokio?/process",
|
||||
"tokio?/rt",
|
||||
"tokio?/sync",
|
||||
"tokio?/time",
|
||||
]
|
||||
full-doc = [
|
||||
"default",
|
||||
"tungstenite",
|
||||
"twilight",
|
||||
"builtin-queue",
|
||||
"receive",
|
||||
]
|
||||
gateway = [
|
||||
"dep:async-trait",
|
||||
"dep:dashmap",
|
||||
"dep:flume",
|
||||
"dep:parking_lot",
|
||||
"dep:tokio",
|
||||
"tokio?/sync",
|
||||
"tokio?/time",
|
||||
]
|
||||
internals = ["dep:byteorder"]
|
||||
native = [
|
||||
"reqwest?/native-tls",
|
||||
"serenity?/native_tls_backend",
|
||||
"stream_lib?/native-tls",
|
||||
"tokio-tungstenite?/native-tls",
|
||||
"tokio-websockets?/native-tls",
|
||||
"twilight-gateway?/native-tls",
|
||||
]
|
||||
receive = [
|
||||
"dep:bytes",
|
||||
"discortp?/demux",
|
||||
"discortp?/rtcp",
|
||||
]
|
||||
rustls = [
|
||||
"reqwest?/rustls-tls",
|
||||
"serenity?/rustls_backend",
|
||||
"stream_lib?/rustls-tls",
|
||||
"tokio-tungstenite?/rustls-tls-webpki-roots",
|
||||
"tokio-websockets?/ring",
|
||||
"tokio-websockets?/rustls-native-roots",
|
||||
"twilight-gateway?/rustls-native-roots",
|
||||
]
|
||||
tungstenite = ["dep:tokio-tungstenite"]
|
||||
twilight = [
|
||||
"dep:twilight-gateway",
|
||||
"dep:twilight-model",
|
||||
]
|
||||
tws = ["dep:tokio-websockets"]
|
||||
|
||||
[lib]
|
||||
name = "songbird"
|
||||
path = "src/lib.rs"
|
||||
bench = false
|
||||
|
||||
[dependencies.aead]
|
||||
version = "0.5.2"
|
||||
optional = true
|
||||
|
||||
[dependencies.aes-gcm]
|
||||
version = "0.10.3"
|
||||
optional = true
|
||||
|
||||
[dependencies.async-trait]
|
||||
version = "0.1"
|
||||
optional = true
|
||||
|
||||
[dependencies.byteorder]
|
||||
version = "1"
|
||||
optional = true
|
||||
|
||||
[dependencies.bytes]
|
||||
version = "1"
|
||||
optional = true
|
||||
|
||||
[dependencies.chacha20poly1305]
|
||||
version = "0.10.1"
|
||||
optional = true
|
||||
|
||||
[dependencies.crypto-common]
|
||||
version = "0.1"
|
||||
features = ["std"]
|
||||
optional = true
|
||||
|
||||
[dependencies.dashmap]
|
||||
version = "6.1.0"
|
||||
optional = true
|
||||
|
||||
[dependencies.davey]
|
||||
version = "0.1.2"
|
||||
optional = true
|
||||
|
||||
[dependencies.derivative]
|
||||
version = "2"
|
||||
|
||||
[dependencies.discortp]
|
||||
version = "0.6"
|
||||
features = [
|
||||
"discord",
|
||||
"pnet",
|
||||
"rtp",
|
||||
]
|
||||
optional = true
|
||||
default-features = false
|
||||
|
||||
[dependencies.either]
|
||||
version = "1.9.0"
|
||||
|
||||
[dependencies.enum-map]
|
||||
version = "2.7.3"
|
||||
|
||||
[dependencies.flume]
|
||||
version = "0.11"
|
||||
optional = true
|
||||
|
||||
[dependencies.futures]
|
||||
version = "0.3"
|
||||
|
||||
[dependencies.nohash-hasher]
|
||||
version = "0.2.0"
|
||||
optional = true
|
||||
|
||||
[dependencies.nonmax]
|
||||
version = "0.5.5"
|
||||
|
||||
[dependencies.opus2]
|
||||
version = "0.4.0"
|
||||
optional = true
|
||||
|
||||
[dependencies.parking_lot]
|
||||
version = "0.12"
|
||||
optional = true
|
||||
|
||||
[dependencies.pin-project]
|
||||
version = "1"
|
||||
|
||||
[dependencies.rand]
|
||||
version = "0.9"
|
||||
optional = true
|
||||
|
||||
[dependencies.reqwest]
|
||||
version = "0.12.2"
|
||||
features = ["stream"]
|
||||
optional = true
|
||||
default-features = false
|
||||
|
||||
[dependencies.ringbuf]
|
||||
version = "0.4"
|
||||
optional = true
|
||||
|
||||
[dependencies.rubato]
|
||||
version = "0.16"
|
||||
optional = true
|
||||
|
||||
[dependencies.rusty_pool]
|
||||
version = "0.7"
|
||||
optional = true
|
||||
|
||||
[dependencies.serde]
|
||||
version = "1"
|
||||
features = ["derive"]
|
||||
|
||||
[dependencies.serde-aux]
|
||||
version = "4"
|
||||
optional = true
|
||||
|
||||
[dependencies.serde_json]
|
||||
version = "1"
|
||||
|
||||
[dependencies.serenity]
|
||||
version = "0.12.0"
|
||||
features = [
|
||||
"voice",
|
||||
"gateway",
|
||||
]
|
||||
optional = true
|
||||
default-features = false
|
||||
|
||||
[dependencies.serenity-voice-model]
|
||||
version = "0.3"
|
||||
optional = true
|
||||
|
||||
[dependencies.socket2]
|
||||
version = "0.5"
|
||||
optional = true
|
||||
|
||||
[dependencies.stream_lib]
|
||||
version = "0.5.2"
|
||||
optional = true
|
||||
default-features = false
|
||||
|
||||
[dependencies.streamcatcher]
|
||||
version = "1"
|
||||
optional = true
|
||||
|
||||
[dependencies.symphonia]
|
||||
version = "0.5.2"
|
||||
optional = true
|
||||
default-features = false
|
||||
|
||||
[dependencies.symphonia-core]
|
||||
version = "0.5.2"
|
||||
optional = true
|
||||
|
||||
[dependencies.tokio]
|
||||
version = "1.0"
|
||||
optional = true
|
||||
default-features = false
|
||||
|
||||
[dependencies.tokio-tungstenite]
|
||||
version = "0.26"
|
||||
features = ["url"]
|
||||
optional = true
|
||||
|
||||
[dependencies.tokio-util]
|
||||
version = "0.7"
|
||||
features = ["io"]
|
||||
optional = true
|
||||
|
||||
[dependencies.tokio-websockets]
|
||||
version = "0.11"
|
||||
features = [
|
||||
"client",
|
||||
"fastrand",
|
||||
"sha1_smol",
|
||||
"simd",
|
||||
]
|
||||
optional = true
|
||||
|
||||
[dependencies.tracing]
|
||||
version = "0.1"
|
||||
features = ["log"]
|
||||
|
||||
[dependencies.tracing-futures]
|
||||
version = "0.2"
|
||||
|
||||
[dependencies.twilight-gateway]
|
||||
version = ">= 0.16.0, <= 0.17"
|
||||
optional = true
|
||||
default-features = false
|
||||
|
||||
[dependencies.twilight-model]
|
||||
version = ">= 0.16.0, <= 0.17"
|
||||
optional = true
|
||||
default-features = false
|
||||
|
||||
[dependencies.typenum]
|
||||
version = "1.17.0"
|
||||
optional = true
|
||||
|
||||
[dependencies.url]
|
||||
version = "2"
|
||||
optional = true
|
||||
|
||||
[dependencies.uuid]
|
||||
version = "1"
|
||||
features = ["v4"]
|
||||
optional = true
|
||||
|
||||
[dev-dependencies.byteorder]
|
||||
version = "1"
|
||||
|
||||
[dev-dependencies.criterion]
|
||||
version = "0.5"
|
||||
|
||||
[dev-dependencies.ntest]
|
||||
version = "0.9"
|
||||
|
||||
[dev-dependencies.symphonia]
|
||||
version = "0.5.2"
|
||||
features = [
|
||||
"aac",
|
||||
"isomp4",
|
||||
"mp3",
|
||||
]
|
||||
|
||||
[dev-dependencies.tokio]
|
||||
version = "1"
|
||||
features = [
|
||||
"rt",
|
||||
"rt-multi-thread",
|
||||
"test-util",
|
||||
]
|
||||
183
app/vendor/songbird/Cargo.toml.orig
generated
vendored
183
app/vendor/songbird/Cargo.toml.orig
generated
vendored
@@ -1,183 +0,0 @@
|
||||
[package]
|
||||
authors = ["Kyle Simpson <kyleandrew.simpson@gmail.com>"]
|
||||
description = "An async Rust library for the Discord voice API."
|
||||
documentation = "https://docs.rs/songbird"
|
||||
edition = "2021"
|
||||
homepage = "https://github.com/serenity-rs/songbird"
|
||||
include = ["src/**/*.rs", "Cargo.toml", "build.rs"]
|
||||
keywords = ["discord", "api", "rtp", "audio"]
|
||||
license = "ISC"
|
||||
name = "songbird"
|
||||
readme = "README.md"
|
||||
repository = "https://github.com/serenity-rs/songbird.git"
|
||||
rust-version = "1.83.0"
|
||||
version = "0.6.1"
|
||||
|
||||
[dependencies]
|
||||
aead = { optional = true, version = "0.5.2" }
|
||||
aes-gcm = { optional = true, version = "0.10.3" }
|
||||
async-trait = { optional = true, version = "0.1" }
|
||||
opus2 = { optional = true, version = "0.4.0" }
|
||||
byteorder = { optional = true, version = "1" }
|
||||
bytes = { optional = true, version = "1" }
|
||||
chacha20poly1305 = { optional = true, version = "0.10.1" }
|
||||
crypto-common = { optional = true, features = ["std"], version = "0.1" }
|
||||
dashmap = { optional = true, version = "6.1.0" }
|
||||
davey = { optional = true, version = "0.1.2" }
|
||||
derivative = "2"
|
||||
discortp = { default-features = false, features = [
|
||||
"discord",
|
||||
"pnet",
|
||||
"rtp",
|
||||
], optional = true, version = "0.6" }
|
||||
either = "1.9.0"
|
||||
flume = { optional = true, version = "0.11" }
|
||||
futures = "0.3"
|
||||
nohash-hasher = { optional = true, version = "0.2.0" }
|
||||
parking_lot = { optional = true, version = "0.12" }
|
||||
pin-project = "1"
|
||||
rand = { optional = true, version = "0.9" }
|
||||
reqwest = { default-features = false, features = [
|
||||
"stream",
|
||||
], optional = true, version = "0.12.2" }
|
||||
ringbuf = { optional = true, version = "0.4" }
|
||||
rubato = { optional = true, version = "0.16" }
|
||||
rusty_pool = { optional = true, version = "0.7" }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde-aux = { optional = true, version = "4" }
|
||||
serde_json = "1"
|
||||
serenity = { default-features = false, optional = true, version = "0.12.0", features = [
|
||||
"voice",
|
||||
"gateway",
|
||||
] }
|
||||
serenity-voice-model = { optional = true, version = "0.3" }
|
||||
socket2 = { optional = true, version = "0.5" }
|
||||
streamcatcher = { optional = true, version = "1" }
|
||||
stream_lib = { default-features = false, optional = true, version = "0.5.2" }
|
||||
symphonia = { default-features = false, optional = true, version = "0.5.2" }
|
||||
symphonia-core = { optional = true, version = "0.5.2" }
|
||||
tokio = { default-features = false, optional = true, version = "1.0" }
|
||||
tokio-tungstenite = { optional = true, version = "0.26", features = ["url"] }
|
||||
tokio-websockets = { optional = true, version = "0.11", features = [
|
||||
"client",
|
||||
"fastrand",
|
||||
"sha1_smol",
|
||||
"simd",
|
||||
] }
|
||||
tokio-util = { features = ["io"], optional = true, version = "0.7" }
|
||||
tracing = { version = "0.1", features = ["log"] }
|
||||
tracing-futures = "0.2"
|
||||
twilight-gateway = { default-features = false, optional = true, version = ">= 0.16.0, <= 0.17" }
|
||||
twilight-model = { default-features = false, optional = true, version = ">= 0.16.0, <= 0.17" }
|
||||
typenum = { optional = true, version = "1.17.0" }
|
||||
url = { optional = true, version = "2" }
|
||||
uuid = { features = ["v4"], optional = true, version = "1" }
|
||||
nonmax = "0.5.5"
|
||||
enum-map = "2.7.3"
|
||||
|
||||
[dev-dependencies]
|
||||
byteorder = "1"
|
||||
criterion = "0.5"
|
||||
ntest = "0.9"
|
||||
symphonia = { version = "0.5.2", features = ["aac", "isomp4", "mp3"] }
|
||||
tokio = { version = "1", features = ["rt", "rt-multi-thread", "test-util"] }
|
||||
|
||||
[features]
|
||||
# Core features
|
||||
default = ["driver", "gateway", "rustls", "serenity", "tungstenite"]
|
||||
gateway = [
|
||||
"dep:async-trait",
|
||||
"dep:dashmap",
|
||||
"dep:flume",
|
||||
"dep:parking_lot",
|
||||
"dep:tokio",
|
||||
"tokio?/sync",
|
||||
"tokio?/time",
|
||||
]
|
||||
driver = [
|
||||
"dep:aead",
|
||||
"dep:aes-gcm",
|
||||
"dep:async-trait",
|
||||
"dep:opus2",
|
||||
"dep:byteorder",
|
||||
"dep:bytes",
|
||||
"dep:chacha20poly1305",
|
||||
"dep:crypto-common",
|
||||
"dep:davey",
|
||||
"dep:discortp",
|
||||
"dep:flume",
|
||||
"dep:nohash-hasher",
|
||||
"dep:parking_lot",
|
||||
"dep:rand",
|
||||
"dep:reqwest",
|
||||
"dep:ringbuf",
|
||||
"dep:rubato",
|
||||
"dep:rusty_pool",
|
||||
"dep:serde-aux",
|
||||
"dep:serenity-voice-model",
|
||||
"dep:socket2",
|
||||
"dep:stream_lib",
|
||||
"dep:streamcatcher",
|
||||
"dep:symphonia",
|
||||
"dep:symphonia-core",
|
||||
"dep:tokio",
|
||||
"dep:tokio-util",
|
||||
"dep:typenum",
|
||||
"dep:url",
|
||||
"dep:uuid",
|
||||
"tokio?/fs",
|
||||
"tokio?/io-util",
|
||||
"tokio?/macros",
|
||||
"tokio?/net",
|
||||
"tokio?/process",
|
||||
"tokio?/rt",
|
||||
"tokio?/sync",
|
||||
"tokio?/time",
|
||||
]
|
||||
rustls = [
|
||||
"reqwest?/rustls-tls",
|
||||
"serenity?/rustls_backend",
|
||||
"stream_lib?/rustls-tls",
|
||||
"tokio-tungstenite?/rustls-tls-webpki-roots",
|
||||
"tokio-websockets?/ring",
|
||||
"tokio-websockets?/rustls-native-roots",
|
||||
"twilight-gateway?/rustls-native-roots",
|
||||
]
|
||||
native = [
|
||||
"reqwest?/native-tls",
|
||||
"serenity?/native_tls_backend",
|
||||
"stream_lib?/native-tls",
|
||||
"tokio-tungstenite?/native-tls",
|
||||
"tokio-websockets?/native-tls",
|
||||
"twilight-gateway?/native-tls",
|
||||
]
|
||||
tungstenite = ["dep:tokio-tungstenite"]
|
||||
tws = ["dep:tokio-websockets"]
|
||||
twilight = ["dep:twilight-gateway", "dep:twilight-model"]
|
||||
|
||||
# Behaviour altering features.
|
||||
builtin-queue = []
|
||||
receive = ["dep:bytes", "discortp?/demux", "discortp?/rtcp"]
|
||||
|
||||
# Used for docgen/testing/benchmarking.
|
||||
full-doc = ["default", "tungstenite", "twilight", "builtin-queue", "receive"]
|
||||
internals = ["dep:byteorder"]
|
||||
|
||||
[lib]
|
||||
bench = false
|
||||
|
||||
[[bench]]
|
||||
name = "base-mixing"
|
||||
path = "benches/base-mixing.rs"
|
||||
required-features = ["internals"]
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mixing-task"
|
||||
path = "benches/mixing-task.rs"
|
||||
required-features = ["internals"]
|
||||
harness = false
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
features = ["full-doc"]
|
||||
rustdoc-args = ["--cfg", "docsrs"]
|
||||
15
app/vendor/songbird/JARVIS-PATCH.md
vendored
15
app/vendor/songbird/JARVIS-PATCH.md
vendored
@@ -1,15 +0,0 @@
|
||||
# songbird 0.6.1, patched for Jarvis
|
||||
|
||||
This is songbird 0.6.1 from crates.io (https://github.com/serenity-rs/songbird, ISC licence, see its
|
||||
README), used through `[patch.crates-io]` in `app/Cargo.toml` and `app/shell/Cargo.toml`.
|
||||
|
||||
The one change: RTP packets of an SSRC that Discord has not named yet are held for up to a second
|
||||
instead of dropped, and processed once its Speaking op names it: before the next packet, or on the
|
||||
next 20 ms playout tick when none follows (`src/driver/tasks/udp_rx/held.rs`, `udp_rx/mod.rs`
|
||||
`process_udp_message` and `release_named`; `ws.rs` sends the Speaking event before naming the SSRC so
|
||||
handlers see the name before the held packets). Discord's Speaking op comes after a speaker's first
|
||||
packets, and unpatched songbird dropped DAVE packets it could not yet decrypt, so the first ~100 ms
|
||||
of the first thing each person said after a join were lost (live 2026-11-18: "자비스 서울 날씨 어때"
|
||||
arrived as "아비 서울날씨 어때").
|
||||
|
||||
Drop this copy when an upstream release does the same.
|
||||
97
app/vendor/songbird/README.md
vendored
97
app/vendor/songbird/README.md
vendored
@@ -1,97 +0,0 @@
|
||||
[![crates.io badge]][crates.io] [![build badge]][build] [![docs badge]][docs] [![current docs badge]][current docs] [![next docs badge]][next docs] [![msrv badge]][msrv] [![license badge]][license] [![guild badge]][guild]
|
||||
|
||||
# Songbird
|
||||
|
||||

|
||||
|
||||
Songbird is an async, cross-library compatible voice system for Discord, written in Rust.
|
||||
The library offers:
|
||||
* A standalone gateway frontend compatible with [serenity] and [twilight] using the
|
||||
`"gateway"` and `"[serenity/twilight]"` plus `"[rustls/native]"` features. You can even run
|
||||
driverless, to help manage your [lavalink] sessions.
|
||||
* A standalone driver for voice calls, via the `"driver"` feature. If you can create
|
||||
a `ConnectionInfo` using any other gateway, or language for your bot, then you
|
||||
can run the songbird voice driver.
|
||||
* Voice receive and RT(C)P packet handling via the `"receive"` feature.
|
||||
* And, by default, a fully featured voice system featuring events, queues,
|
||||
seeking on compatible streams, shared multithreaded audio stream caches,
|
||||
and direct Opus data passthrough from DCA files.
|
||||
|
||||
## Intents
|
||||
Songbird's gateway functionality requires you to specify the `GUILD_VOICE_STATES` intent.
|
||||
|
||||
## Codec support
|
||||
Songbird supports all [codecs and formats provided by Symphonia] (pure-Rust), with Opus support
|
||||
provided by [opus2] (an FFI wrapper for libopus).
|
||||
|
||||
**By default, *Songbird will not request any codecs from Symphonia*.** To change this, in your own
|
||||
project you will need to depend on Symphonia as well.
|
||||
|
||||
```toml
|
||||
# Including songbird alone gives you support for Opus via the DCA file format.
|
||||
[dependencies.songbird]
|
||||
version = "0.5"
|
||||
features = ["builtin-queue"]
|
||||
|
||||
# To get additional codecs, you *must* add Symphonia yourself.
|
||||
# This includes the default formats (MKV/WebM, Ogg, Wave) and codecs (FLAC, PCM, Vorbis)...
|
||||
[dependencies.symphonia]
|
||||
version = "0.5"
|
||||
features = ["aac", "mp3", "isomp4", "alac"] # ...as well as any extras you need!
|
||||
```
|
||||
|
||||
## Dependencies
|
||||
Songbird needs a few system dependencies before you can use it.
|
||||
|
||||
- Opus - Audio codec that Discord uses.
|
||||
[opus2] will use installed libopus binaries if available via pkgconf on Linux/MacOS, otherwise you will need to install cmake to build opus from source.
|
||||
This is always the case on Windows.
|
||||
For Unix systems, you can install the library with `apt install libopus-dev` on Ubuntu or `pacman -S opus` on Arch Linux.
|
||||
If you do not have it installed it will be built for you. However, you will need a C compiler and the GNU autotools installed.
|
||||
Again, these can be installed with `apt install build-essential autoconf automake libtool m4` on Ubuntu or `pacman -S base-devel` on Arch Linux.
|
||||
|
||||
This is a required dependency. Songbird cannot work without it.
|
||||
|
||||
- yt-dlp / youtube-dl / (similar forks) - Audio/Video download tool.
|
||||
yt-dlp can be installed [according to the installation instructions on the main repo].
|
||||
You can install youtube-dl with Python's package manager, pip, which we recommend for youtube-dl. You can do it with the command `pip install youtube_dl`.
|
||||
Alternatively, you can install it with your system's package manager, `apt install youtube-dl` on Ubuntu or `pacman -S youtube-dl` on Arch Linux.
|
||||
|
||||
This is an optional dependency for users, but is required as a dev-dependency. It allows Songbird to download audio/video sources from the Internet from a variety of webpages, which it will convert to the Opus audio format Discord uses.
|
||||
|
||||
## Examples
|
||||
Full examples showing various types of functionality and integrations can be found in [this crate's examples directory].
|
||||
|
||||
## Contributing
|
||||
If you want to help out or file an issue, please look over [our contributor guidelines]!
|
||||
|
||||
## Attribution
|
||||
Songbird's logo is based upon the copyright-free image ["Black-Capped Chickadee"] by George Gorgas White.
|
||||
|
||||
[serenity]: https://github.com/serenity-rs/serenity
|
||||
[twilight]: https://github.com/twilight-rs/twilight
|
||||
["Black-Capped Chickadee"]: https://www.oldbookillustrations.com/illustrations/black-capped-chickadee/
|
||||
[lavalink]: https://github.com/freyacodes/Lavalink
|
||||
[this crate's examples directory]: https://github.com/serenity-rs/songbird/tree/current/examples
|
||||
[our contributor guidelines]: CONTRIBUTING.md
|
||||
[codecs and formats provided by Symphonia]: https://github.com/pdeljanov/Symphonia#formats-demuxers
|
||||
[opus2]: https://github.com/cijiugechu/opus2
|
||||
[according to the installation instructions on the main repo]: https://github.com/yt-dlp/yt-dlp#installation
|
||||
|
||||
<!-- Badges -->
|
||||
[crates.io]: https://crates.io/crates/songbird
|
||||
[crates.io badge]: https://img.shields.io/crates/v/songbird.svg?style=flat-square
|
||||
[build]: https://github.com/serenity-rs/songbird/actions
|
||||
[build badge]: https://img.shields.io/github/actions/workflow/status/serenity-rs/songbird/ci.yml?branch=current&style=flat-square
|
||||
[docs]: https://docs.rs/songbird/
|
||||
[docs badge]: https://img.shields.io/badge/docs-online-informational?style=flat-square
|
||||
[current docs]: https://serenity-rs.github.io/songbird/current/songbird/index.html
|
||||
[current docs badge]: https://img.shields.io/badge/docs-current-4d76ae.svg?style=flat-square
|
||||
[next docs]: https://serenity-rs.github.io/songbird/next/songbird/index.html
|
||||
[next docs badge]: https://img.shields.io/badge/docs-next-4d76ae.svg?style=flat-square
|
||||
[msrv]: https://blog.rust-lang.org/2024/11/28/Rust-1.83.0.html
|
||||
[msrv badge]: https://img.shields.io/badge/rust-1.83+-93450a.svg?style=flat-square
|
||||
[license]: LICENSE
|
||||
[license badge]: https://img.shields.io/crates/l/songbird.svg?style=flat-square&color=yellow
|
||||
[guild]: https://discord.gg/serenity-rs
|
||||
[guild badge]: https://img.shields.io/discord/381880193251409931.svg?style=flat-square&colorB=7289DA
|
||||
24
app/vendor/songbird/build.rs
vendored
24
app/vendor/songbird/build.rs
vendored
@@ -1,24 +0,0 @@
|
||||
#[cfg(all(feature = "driver", not(any(feature = "rustls", feature = "native"))))]
|
||||
compile_error!(
|
||||
"You have the `driver` feature enabled: \
|
||||
either the `rustls` or `native` feature must be
|
||||
selected to let Songbird's driver use websockets.\n\
|
||||
- `rustls` uses Rustls, a pure Rust TLS-implemenation.\n\
|
||||
- `native` uses SChannel on Windows, Secure Transport on macOS, \
|
||||
and OpenSSL on other platforms.\n\
|
||||
If you are unsure, go with `rustls`."
|
||||
);
|
||||
|
||||
#[cfg(any(
|
||||
all(feature = "driver", feature = "tws", feature = "tungstenite"),
|
||||
all(feature = "driver", not(feature = "tws"), not(feature = "tungstenite"))
|
||||
))]
|
||||
compile_error!(
|
||||
"You have the `driver` feature enabled: \
|
||||
this requires you specify either: \n\
|
||||
- `tungstenite` (recommended with serenity)\n\
|
||||
- or `tws` (recommended with twilight).\n\
|
||||
You have either specified none, or both - choose exactly one."
|
||||
);
|
||||
|
||||
fn main() {}
|
||||
444
app/vendor/songbird/src/config.rs
vendored
444
app/vendor/songbird/src/config.rs
vendored
@@ -1,444 +0,0 @@
|
||||
#[cfg(all(feature = "driver", feature = "receive"))]
|
||||
use crate::driver::DecodeMode;
|
||||
use crate::FloatDuration;
|
||||
#[cfg(feature = "driver")]
|
||||
use crate::{
|
||||
driver::{
|
||||
get_default_scheduler,
|
||||
retry::Retry,
|
||||
tasks::disposal::DisposalThread,
|
||||
CryptoMode,
|
||||
MixMode,
|
||||
Scheduler,
|
||||
},
|
||||
input::codecs::*,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
use crate::driver::test_config::*;
|
||||
#[cfg(all(test, feature = "driver"))]
|
||||
use crate::driver::SchedulerConfig;
|
||||
|
||||
#[cfg(feature = "driver")]
|
||||
use symphonia::core::{codecs::CodecRegistry, probe::Probe};
|
||||
|
||||
use derivative::Derivative;
|
||||
#[cfg(all(feature = "driver", feature = "receive"))]
|
||||
use std::num::NonZeroU8;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Configuration for drivers and calls.
|
||||
#[derive(Clone, Derivative)]
|
||||
#[derivative(Debug)]
|
||||
#[non_exhaustive]
|
||||
pub struct Config {
|
||||
#[cfg(feature = "driver")]
|
||||
/// Selected tagging mode for voice packet encryption.
|
||||
///
|
||||
/// Defaults to [`CryptoMode::Aes256Gcm`].
|
||||
///
|
||||
/// Changes to this field will not immediately apply if the
|
||||
/// driver is actively connected, but will apply to subsequent
|
||||
/// sessions.
|
||||
///
|
||||
/// [`CryptoMode::Aes256Gcm`]: CryptoMode::Aes256Gcm
|
||||
pub crypto_mode: CryptoMode,
|
||||
|
||||
#[cfg(all(feature = "driver", feature = "receive"))]
|
||||
/// Configures whether decoding and decryption occur for all received packets.
|
||||
///
|
||||
/// If receiving and using voice packets, generally you should choose [`DecodeMode::Decode`].
|
||||
/// [`DecodeMode::Decrypt`] is intended for users running their own selective decoding or
|
||||
/// who need to inspect Opus packets. [User speaking state] can still be seen using [`DecodeMode::Pass`].
|
||||
/// If you're certain you will never need any RT(C)P events, then consider building without
|
||||
/// the `"receive"` feature for extra performance.
|
||||
///
|
||||
/// Defaults to [`DecodeMode::Decrypt`]. This is due to per-packet decoding costs,
|
||||
/// which most users will not want to pay, but allowing speaking events which are commonly used.
|
||||
///
|
||||
/// [`DecodeMode::Decode`]: DecodeMode::Decode
|
||||
/// [`DecodeMode::Decrypt`]: DecodeMode::Decrypt
|
||||
/// [`DecodeMode::Pass`]: DecodeMode::Pass
|
||||
/// [User speaking state]: crate::events::CoreEvent::VoiceTick
|
||||
pub decode_mode: DecodeMode,
|
||||
|
||||
#[cfg(all(feature = "driver", feature = "receive"))]
|
||||
/// Configures the amount of time after a user/SSRC is inactive before their decoder state
|
||||
/// should be removed.
|
||||
///
|
||||
/// Defaults to 1 minute.
|
||||
pub decode_state_timeout: FloatDuration,
|
||||
|
||||
#[cfg(all(feature = "driver", feature = "receive"))]
|
||||
/// Configures the number of audio packets to buffer for each user before playout.
|
||||
///
|
||||
/// A playout buffer allows Songbird to smooth out jitter in audio packet arrivals,
|
||||
/// as well as to correct for reordering of packets by the network.
|
||||
///
|
||||
/// This does not affect the arrival of raw packet events.
|
||||
///
|
||||
/// Defaults to 5 packets (100ms).
|
||||
pub playout_buffer_length: NonZeroU8,
|
||||
|
||||
#[cfg(all(feature = "driver", feature = "receive"))]
|
||||
/// Configures the initial amount of extra space allocated to handle packet bursts.
|
||||
///
|
||||
/// Each SSRC's receive buffer will start at capacity `playout_buffer_length +
|
||||
/// playout_spike_length`, up to a maximum 64 packets.
|
||||
///
|
||||
/// Defaults to 3 packets (thus capacity defaults to 8).
|
||||
pub playout_spike_length: u8,
|
||||
|
||||
#[cfg(feature = "gateway")]
|
||||
/// Configures the amount of time to wait for Discord to reply with connection information
|
||||
/// if [`Call::join`]/[`join_gateway`] are used.
|
||||
///
|
||||
/// This is a useful fallback in the event that:
|
||||
/// * the underlying Discord client restarts and loses a join request, or
|
||||
/// * a channel join fails because the bot is already believed to be there.
|
||||
///
|
||||
/// Defaults to 10 seconds. If set to `None`, connections will never time out.
|
||||
///
|
||||
/// [`Call::join`]: crate::Call::join
|
||||
/// [`join_gateway`]: crate::Call::join_gateway
|
||||
pub gateway_timeout: Option<FloatDuration>,
|
||||
|
||||
#[cfg(feature = "driver")]
|
||||
/// Configures whether the driver will mix and output stereo or mono Opus data
|
||||
/// over a voice channel.
|
||||
///
|
||||
/// Defaults to [`Stereo`].
|
||||
///
|
||||
/// [`Stereo`]: MixMode::Stereo
|
||||
pub mix_mode: MixMode,
|
||||
|
||||
#[cfg(feature = "driver")]
|
||||
/// Number of concurrently active tracks to allocate memory for.
|
||||
///
|
||||
/// This should be set at, or just above, the maximum number of tracks
|
||||
/// you expect your bot will play at the same time. Exceeding the size of
|
||||
/// the internal queue will trigger a larger memory allocation and copy,
|
||||
/// possibly causing the mixer thread to miss a packet deadline.
|
||||
///
|
||||
/// Defaults to `1`.
|
||||
///
|
||||
/// Changes to this field in a running driver will only ever increase
|
||||
/// the capacity of the track store.
|
||||
pub preallocated_tracks: u8,
|
||||
|
||||
#[cfg(feature = "driver")]
|
||||
/// Connection retry logic for the [`Driver`].
|
||||
///
|
||||
/// This controls how many times the [`Driver`] should retry any connections,
|
||||
/// as well as how long to wait between attempts.
|
||||
///
|
||||
/// [`Driver`]: crate::driver::Driver
|
||||
pub driver_retry: Retry,
|
||||
|
||||
#[cfg(feature = "driver")]
|
||||
/// Configures whether or not each mixed audio packet is [soft-clipped] into the
|
||||
/// [-1, 1] audio range.
|
||||
///
|
||||
/// Defaults to `true`, preventing clipping and dangerously loud audio from being sent.
|
||||
///
|
||||
/// **This operation adds ~3% cost to a standard (non-passthrough) mix cycle.**
|
||||
/// If you *know* that your bot will only play one sound at a time and that
|
||||
/// your volume is between `0.0` and `1.0`, then you can disable soft-clipping
|
||||
/// for a performance boost. If you are playing several sounds at once, do not
|
||||
/// disable this unless you make sure to reduce the volume of each sound.
|
||||
///
|
||||
/// [soft-clipped]: https://opus-codec.org/docs/opus_api-1.3.1/group__opus__decoder.html#gaff99598b352e8939dded08d96e125e0b
|
||||
pub use_softclip: bool,
|
||||
|
||||
#[cfg(feature = "driver")]
|
||||
/// Configures the maximum amount of time to wait for an attempted voice
|
||||
/// connection to Discord.
|
||||
///
|
||||
/// Defaults to 10 seconds. If set to `None`, connections will never time out.
|
||||
pub driver_timeout: Option<FloatDuration>,
|
||||
|
||||
#[cfg(feature = "driver")]
|
||||
#[derivative(Debug = "ignore")]
|
||||
/// Registry of the inner codecs supported by the driver, adding opus2-based
|
||||
/// Opus codec support to all of Symphonia's default codecs.
|
||||
///
|
||||
/// Defaults to [`get_codec_registry`].
|
||||
pub codec_registry: &'static CodecRegistry,
|
||||
|
||||
#[cfg(feature = "driver")]
|
||||
#[derivative(Debug = "ignore")]
|
||||
/// Registry of the muxers and container formats supported by the driver.
|
||||
///
|
||||
/// Defaults to [`get_probe`], which includes all of Symphonia's default format handlers
|
||||
/// and DCA format support.
|
||||
pub format_registry: &'static Probe,
|
||||
|
||||
#[cfg(feature = "driver")]
|
||||
/// The Sender for a channel that will run the destructor of possibly blocking values.
|
||||
///
|
||||
/// If not set, a thread will be spawned to perform this, but it is recommended to create
|
||||
/// a long running thread instead of relying on a per-driver thread.
|
||||
///
|
||||
/// Note: When using [`Songbird`] this is overwritten automatically by its disposal thread.
|
||||
///
|
||||
/// [`Songbird`]: crate::Songbird
|
||||
pub disposer: Option<DisposalThread>,
|
||||
|
||||
#[cfg(feature = "driver")]
|
||||
/// The scheduler is responsible for mapping idle and active [`Driver`] instances
|
||||
/// to threads.
|
||||
///
|
||||
/// If set to None, then songbird will use [`get_default_scheduler`].
|
||||
///
|
||||
/// [`Driver`]: crate::Driver
|
||||
pub scheduler: Option<Scheduler>,
|
||||
|
||||
// Test only attributes
|
||||
#[cfg(feature = "driver")]
|
||||
#[cfg(test)]
|
||||
/// Test config to offer precise control over mixing tick rate/count.
|
||||
pub(crate) tick_style: TickStyle,
|
||||
#[cfg(feature = "driver")]
|
||||
#[cfg(test)]
|
||||
/// If set, skip connection and encryption steps.
|
||||
pub(crate) override_connection: Option<OutputMode>,
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
#[cfg(feature = "driver")]
|
||||
crypto_mode: CryptoMode::Aes256Gcm,
|
||||
#[cfg(all(feature = "driver", feature = "receive"))]
|
||||
decode_mode: DecodeMode::Decrypt,
|
||||
#[cfg(all(feature = "driver", feature = "receive"))]
|
||||
decode_state_timeout: Duration::from_secs(60).into(),
|
||||
#[cfg(all(feature = "driver", feature = "receive"))]
|
||||
playout_buffer_length: NonZeroU8::new(5).unwrap(),
|
||||
#[cfg(all(feature = "driver", feature = "receive"))]
|
||||
playout_spike_length: 3,
|
||||
#[cfg(feature = "gateway")]
|
||||
gateway_timeout: Some(Duration::from_secs(10).into()),
|
||||
#[cfg(feature = "driver")]
|
||||
mix_mode: MixMode::Stereo,
|
||||
#[cfg(feature = "driver")]
|
||||
preallocated_tracks: 1,
|
||||
#[cfg(feature = "driver")]
|
||||
use_softclip: true,
|
||||
#[cfg(feature = "driver")]
|
||||
driver_retry: Retry::default(),
|
||||
#[cfg(feature = "driver")]
|
||||
driver_timeout: Some(Duration::from_secs(10).into()),
|
||||
#[cfg(feature = "driver")]
|
||||
codec_registry: get_codec_registry(),
|
||||
#[cfg(feature = "driver")]
|
||||
format_registry: get_probe(),
|
||||
#[cfg(feature = "driver")]
|
||||
disposer: None,
|
||||
#[cfg(feature = "driver")]
|
||||
scheduler: None,
|
||||
#[cfg(feature = "driver")]
|
||||
#[cfg(test)]
|
||||
tick_style: TickStyle::Timed,
|
||||
#[cfg(feature = "driver")]
|
||||
#[cfg(test)]
|
||||
override_connection: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "driver")]
|
||||
impl Config {
|
||||
/// Sets this `Config`'s chosen cryptographic tagging scheme.
|
||||
#[must_use]
|
||||
pub fn crypto_mode(mut self, crypto_mode: CryptoMode) -> Self {
|
||||
self.crypto_mode = crypto_mode;
|
||||
self
|
||||
}
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
/// Sets this `Config`'s received packet decryption/decoding behaviour.
|
||||
#[must_use]
|
||||
pub fn decode_mode(mut self, decode_mode: DecodeMode) -> Self {
|
||||
self.decode_mode = decode_mode;
|
||||
self
|
||||
}
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
/// Sets this `Config`'s received packet decoder cleanup timer.
|
||||
#[must_use]
|
||||
pub fn decode_state_timeout(mut self, decode_state_timeout: Duration) -> Self {
|
||||
self.decode_state_timeout = decode_state_timeout.into();
|
||||
self
|
||||
}
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
/// Sets this `Config`'s playout buffer length, in packets.
|
||||
#[must_use]
|
||||
pub fn playout_buffer_length(mut self, playout_buffer_length: NonZeroU8) -> Self {
|
||||
self.playout_buffer_length = playout_buffer_length;
|
||||
self
|
||||
}
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
/// Sets this `Config`'s additional pre-allocated space to handle bursty audio packets.
|
||||
#[must_use]
|
||||
pub fn playout_spike_length(mut self, playout_spike_length: u8) -> Self {
|
||||
self.playout_spike_length = playout_spike_length;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets this `Config`'s audio mixing channel count.
|
||||
#[must_use]
|
||||
pub fn mix_mode(mut self, mix_mode: MixMode) -> Self {
|
||||
self.mix_mode = mix_mode;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets this `Config`'s number of tracks to preallocate.
|
||||
#[must_use]
|
||||
pub fn preallocated_tracks(mut self, preallocated_tracks: u8) -> Self {
|
||||
self.preallocated_tracks = preallocated_tracks;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets this `Config`'s number to enable/disable soft-clipping sent audio.
|
||||
#[must_use]
|
||||
pub fn use_softclip(mut self, use_softclip: bool) -> Self {
|
||||
self.use_softclip = use_softclip;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets this `Config`'s timeout for establishing a voice connection.
|
||||
#[must_use]
|
||||
pub fn driver_timeout(mut self, driver_timeout: Option<Duration>) -> Self {
|
||||
self.driver_timeout = driver_timeout.map(Into::into);
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets this `Config`'s voice connection retry configuration.
|
||||
#[must_use]
|
||||
pub fn driver_retry(mut self, driver_retry: Retry) -> Self {
|
||||
self.driver_retry = driver_retry;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets this `Config`'s symphonia codec registry.
|
||||
#[must_use]
|
||||
pub fn codec_registry(mut self, codec_registry: &'static CodecRegistry) -> Self {
|
||||
self.codec_registry = codec_registry;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets this `Config`'s symphonia format registry/probe set.
|
||||
#[must_use]
|
||||
pub fn format_registry(mut self, format_registry: &'static Probe) -> Self {
|
||||
self.format_registry = format_registry;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets this `Config`'s channel for sending disposal messages.
|
||||
#[must_use]
|
||||
pub fn disposer(mut self, disposer: DisposalThread) -> Self {
|
||||
self.disposer = Some(disposer);
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets this `Config`'s mixer scheduler.
|
||||
#[must_use]
|
||||
pub fn scheduler(mut self, scheduler: Scheduler) -> Self {
|
||||
self.scheduler = Some(scheduler);
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns a lightweight reference to the audio scheduler this `Config` will use.
|
||||
#[must_use]
|
||||
pub fn get_scheduler(&self) -> Scheduler {
|
||||
self.scheduler
|
||||
.as_ref()
|
||||
.unwrap_or(get_default_scheduler())
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// Ensures a global disposer has been set, initializing one if not.
|
||||
#[must_use]
|
||||
pub(crate) fn initialise_disposer(self) -> Self {
|
||||
if self.disposer.is_some() {
|
||||
self
|
||||
} else {
|
||||
self.disposer(DisposalThread::run())
|
||||
}
|
||||
}
|
||||
|
||||
/// This is used to prevent changes which would invalidate the current session.
|
||||
pub(crate) fn make_safe(&mut self, previous: &Config, connected: bool) {
|
||||
if connected {
|
||||
self.crypto_mode = previous.crypto_mode;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "driver"))]
|
||||
impl Config {
|
||||
pub(crate) fn initialise_disposer(self) -> Self {
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
// Test only attributes
|
||||
#[cfg(all(test, feature = "driver"))]
|
||||
#[allow(missing_docs)]
|
||||
impl Config {
|
||||
#[must_use]
|
||||
pub fn tick_style(mut self, tick_style: TickStyle) -> Self {
|
||||
self.tick_style = tick_style;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets this `Config`'s voice connection retry configuration.
|
||||
#[must_use]
|
||||
pub fn override_connection(mut self, override_connection: Option<OutputMode>) -> Self {
|
||||
self.override_connection = override_connection;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn test_cfg(raw_output: bool) -> (DriverTestHandle, Config) {
|
||||
let (tick_tx, tick_rx) = flume::unbounded();
|
||||
|
||||
let (conn, rx) = if raw_output {
|
||||
let (pkt_tx, pkt_rx) = flume::unbounded();
|
||||
|
||||
(OutputMode::Raw(pkt_tx), OutputReceiver::Raw(pkt_rx))
|
||||
} else {
|
||||
let (rtp_tx, rtp_rx) = flume::unbounded();
|
||||
|
||||
(OutputMode::Rtp(rtp_tx), OutputReceiver::Rtp(rtp_rx))
|
||||
};
|
||||
|
||||
let sc_config = SchedulerConfig {
|
||||
strategy: crate::driver::SchedulerMode::MaxPerThread(1.try_into().unwrap()),
|
||||
move_expensive_tasks: true,
|
||||
};
|
||||
|
||||
let config = Config::default()
|
||||
.tick_style(TickStyle::UntimedWithExecLimit(tick_rx))
|
||||
// give each test its own thread in the scheduler for simplicity.
|
||||
.scheduler(Scheduler::new(sc_config))
|
||||
.override_connection(Some(conn));
|
||||
|
||||
let handle = DriverTestHandle { rx, tx: tick_tx };
|
||||
|
||||
(handle, config)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "gateway")]
|
||||
impl Config {
|
||||
/// Sets this `Config`'s timeout for joining a voice channel.
|
||||
#[must_use]
|
||||
pub fn gateway_timeout(mut self, gateway_timeout: Option<Duration>) -> Self {
|
||||
self.gateway_timeout = gateway_timeout.map(Into::into);
|
||||
self
|
||||
}
|
||||
}
|
||||
136
app/vendor/songbird/src/constants.rs
vendored
136
app/vendor/songbird/src/constants.rs
vendored
@@ -1,136 +0,0 @@
|
||||
//! Constants affecting driver function and API handling.
|
||||
|
||||
#[cfg(feature = "driver")]
|
||||
use discortp::rtp::RtpType;
|
||||
#[cfg(feature = "driver")]
|
||||
use opus2::Bitrate;
|
||||
use std::time::Duration;
|
||||
|
||||
#[cfg(feature = "driver")]
|
||||
/// The voice gateway version used by the library.
|
||||
pub const VOICE_GATEWAY_VERSION: u8 = crate::model::constants::GATEWAY_VERSION;
|
||||
|
||||
#[cfg(feature = "driver")]
|
||||
/// Sample rate of audio to be sent to Discord.
|
||||
pub const SAMPLE_RATE: u32 = 48000;
|
||||
|
||||
/// Sample rate of audio to be sent to Discord.
|
||||
pub const SAMPLE_RATE_RAW: usize = 48_000;
|
||||
|
||||
/// Number of audio frames/packets to be sent per second.
|
||||
pub const AUDIO_FRAME_RATE: usize = 50;
|
||||
|
||||
/// Length of time between any two audio frames.
|
||||
pub const TIMESTEP_LENGTH: Duration = Duration::from_millis(1000 / AUDIO_FRAME_RATE as u64);
|
||||
|
||||
#[cfg(feature = "driver")]
|
||||
/// Default bitrate for audio.
|
||||
pub const DEFAULT_BITRATE: Bitrate = Bitrate::Bits(128_000);
|
||||
|
||||
/// Number of output samples at 48kHZ to produced when resampling subframes.
|
||||
pub(crate) const RESAMPLE_OUTPUT_FRAME_SIZE: usize = MONO_FRAME_SIZE / 2;
|
||||
|
||||
/// The maximum number of bad frames to allow in an Opus source before blocking passthrough.
|
||||
pub(crate) const OPUS_PASSTHROUGH_STRIKE_LIMIT: u8 = 3;
|
||||
|
||||
/// Number of samples in one complete frame of audio per channel.
|
||||
///
|
||||
/// This is equally the number of stereo (joint) samples in an audio frame.
|
||||
pub const MONO_FRAME_SIZE: usize = SAMPLE_RATE_RAW / AUDIO_FRAME_RATE;
|
||||
|
||||
/// Number of individual samples in one complete frame of stereo audio.
|
||||
pub const STEREO_FRAME_SIZE: usize = 2 * MONO_FRAME_SIZE;
|
||||
|
||||
/// Number of bytes in one complete frame of raw `f32`-encoded mono audio.
|
||||
pub const MONO_FRAME_BYTE_SIZE: usize = MONO_FRAME_SIZE * std::mem::size_of::<f32>();
|
||||
|
||||
/// Number of bytes in one complete frame of raw `f32`-encoded stereo audio.
|
||||
pub const STEREO_FRAME_BYTE_SIZE: usize = STEREO_FRAME_SIZE * std::mem::size_of::<f32>();
|
||||
|
||||
/// Length (in milliseconds) of any audio frame.
|
||||
pub const FRAME_LEN_MS: usize = 1000 / AUDIO_FRAME_RATE;
|
||||
|
||||
/// Maximum number of audio frames/packets to be sent per second to be buffered.
|
||||
pub const CHILD_BUFFER_LEN: usize = AUDIO_FRAME_RATE / 2;
|
||||
|
||||
/// Maximum packet size for a voice packet.
|
||||
///
|
||||
/// Set a safe amount below the Ethernet MTU to avoid fragmentation/rejection.
|
||||
pub const VOICE_PACKET_MAX: usize = 1460;
|
||||
|
||||
/// Delay between sends of UDP keepalive frames.
|
||||
///
|
||||
/// Passive monitoring of Discord itself shows that these fire every 5 seconds
|
||||
/// irrespective of outgoing UDP traffic.
|
||||
pub const UDP_KEEPALIVE_GAP_MS: u64 = 5_000;
|
||||
|
||||
/// Type-converted delay between sends of UDP keepalive frames.
|
||||
///
|
||||
/// Passive monitoring of Discord itself shows that these fire every 5 seconds
|
||||
/// irrespective of outgoing UDP traffic.
|
||||
pub const UDP_KEEPALIVE_GAP: Duration = Duration::from_millis(UDP_KEEPALIVE_GAP_MS);
|
||||
|
||||
/// Opus silent frame, used to signal speech start and end (and prevent audio glitching).
|
||||
pub const SILENT_FRAME: [u8; 3] = [0xf8, 0xff, 0xfe];
|
||||
|
||||
/// The one (and only) RTP version.
|
||||
pub const RTP_VERSION: u8 = 2;
|
||||
|
||||
#[cfg(feature = "driver")]
|
||||
/// Profile type used by Discord's Opus audio traffic.
|
||||
pub const RTP_PROFILE_TYPE: RtpType = RtpType::Dynamic(120);
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
/// The magic marker appended to the end of DAVE encrypted payloads, used to determine
|
||||
/// whether the voice packet is encrypted.
|
||||
pub const DAVE_MAGIC_MARKER: [u8; 2] = [0xfa, 0xfa];
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::doc_markdown, missing_docs)]
|
||||
pub mod test_data {
|
||||
/// URL for a source which YTDL must extract.
|
||||
///
|
||||
/// Referenced under CC BY-NC-SA 3.0 -- https://creativecommons.org/licenses/by-nc-sa/3.0/
|
||||
pub const YTDL_TARGET: &str = "https://cloudkicker.bandcamp.com/track/94-days";
|
||||
|
||||
/// URL for a source that has both a playlist and a music video,
|
||||
/// which YTDL should extract.
|
||||
///
|
||||
/// Referenced under CC BY-NC-SA 3.0 -- https://creativecommons.org/licenses/by-nc-sa/3.0/
|
||||
pub const YTDL_PLAYLIST_TARGET: &str =
|
||||
"https://www.youtube.com/watch?v=KSgEFfWZ-W0&list=OLAK5uy_l2x81ffbpevMSjUn7NniL_rNLulWM3n6g&index=7";
|
||||
|
||||
/// URL for a source which can be read via an Http Request.
|
||||
///
|
||||
/// Referenced under CC BY-NC-SA 3.0 -- https://creativecommons.org/licenses/by-nc-sa/3.0/
|
||||
pub const HTTP_TARGET: &str = "https://github.com/FelixMcFelix/songbird/raw/symphonia/resources/Cloudkicker%20-%202011%2007.mp3";
|
||||
|
||||
/// URL for an opus/ogg source which can be read via an Http Request.
|
||||
///
|
||||
/// Referenced under CC BY 3.0 -- https://creativecommons.org/licenses/by/3.0/
|
||||
pub const HTTP_OPUS_TARGET: &str = "https://github.com/FelixMcFelix/songbird/raw/symphonia/resources/Cloudkicker%20-%20Making%20Will%20Mad.opus";
|
||||
|
||||
/// URL for an opus/webm source which can be read via an Http Request.
|
||||
///
|
||||
/// Referenced under CC BY 3.0 -- https://creativecommons.org/licenses/by/3.0/
|
||||
pub const HTTP_WEBM_TARGET: &str = "https://github.com/FelixMcFelix/songbird/raw/symphonia/resources/Cloudkicker%20-%20Making%20Will%20Mad.webm";
|
||||
|
||||
/// Path to a DCA source.
|
||||
///
|
||||
/// Referenced under CC BY-NC-SA 3.0 -- https://creativecommons.org/licenses/by-nc-sa/3.0/
|
||||
pub const FILE_DCA_TARGET: &str = "resources/Cloudkicker - 2011 07.dca1";
|
||||
|
||||
/// Path to an opus source which can be read via a File.
|
||||
///
|
||||
/// Referenced under CC BY 3.0 -- https://creativecommons.org/licenses/by/3.0/
|
||||
pub const FILE_WEBM_TARGET: &str = "resources/Cloudkicker - Making Will Mad.webm";
|
||||
|
||||
/// Path to a Wav source which can be read via a File.
|
||||
pub const FILE_WAV_TARGET: &str = "resources/loop.wav";
|
||||
|
||||
/// Path to a shorter MP3 source which can be read via a File.
|
||||
pub const FILE_SHORT_MP3_TARGET: &str = "resources/ting.mp3";
|
||||
|
||||
/// Path to an MP4 (H264 + AAC) source which can be read via a File.
|
||||
pub const FILE_VID_TARGET: &str = "resources/ting-vid.mp4";
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
//! Various driver internals which need to be exported for benchmarking.
|
||||
//!
|
||||
//! Included if using the `"internals"` feature flag.
|
||||
//! You should not and/or cannot use these as part of a normal application.
|
||||
|
||||
#![allow(missing_docs)]
|
||||
|
||||
pub use super::tasks::{message as task_message, mixer};
|
||||
|
||||
pub use super::crypto::{Cipher, CryptoState};
|
||||
|
||||
use crate::{
|
||||
driver::tasks::message::TrackContext,
|
||||
tracks::{Track, TrackHandle},
|
||||
};
|
||||
|
||||
#[must_use]
|
||||
pub fn track_context(t: Track) -> (TrackHandle, TrackContext) {
|
||||
t.into_context()
|
||||
}
|
||||
|
||||
pub mod scheduler {
|
||||
pub use crate::driver::scheduler::*;
|
||||
}
|
||||
152
app/vendor/songbird/src/driver/connection/error.rs
vendored
152
app/vendor/songbird/src/driver/connection/error.rs
vendored
@@ -1,152 +0,0 @@
|
||||
//! Connection errors and convenience types.
|
||||
|
||||
use crate::{
|
||||
driver::tasks::{error::Recipient, message::*},
|
||||
ws::Error as WsError,
|
||||
};
|
||||
use aes_gcm::Error as CryptoError;
|
||||
use davey::errors::{
|
||||
CreateKeyPackageError as DaveyCreateKeyPackageError,
|
||||
InitError as DaveyInitError,
|
||||
};
|
||||
use flume::SendError;
|
||||
use serde_json::Error as JsonError;
|
||||
use std::{error::Error as StdError, fmt, io::Error as IoError};
|
||||
use tokio::time::error::Elapsed;
|
||||
|
||||
/// Errors encountered while connecting to a Discord voice server over the driver.
|
||||
#[derive(Debug)]
|
||||
#[non_exhaustive]
|
||||
pub enum Error {
|
||||
/// The driver hung up an internal signaller, either due to another connection attempt
|
||||
/// or a crash.
|
||||
AttemptDiscarded,
|
||||
/// An error occurred during [en/de]cryption of voice packets.
|
||||
Crypto(CryptoError),
|
||||
/// The symmetric key supplied by Discord had the wrong size.
|
||||
CryptoInvalidLength,
|
||||
/// Server did not return the expected crypto mode during negotiation.
|
||||
CryptoModeInvalid,
|
||||
/// Selected crypto mode was not offered by server.
|
||||
CryptoModeUnavailable,
|
||||
/// Failed to create a DAVE key package.
|
||||
#[cfg(feature = "driver")]
|
||||
DaveCreateKeyPackageError(DaveyCreateKeyPackageError),
|
||||
/// An error occured during initialization of the DAVE session.
|
||||
#[cfg(feature = "driver")]
|
||||
DaveInitializationError(DaveyInitError),
|
||||
/// An indicator that an endpoint URL was invalid.
|
||||
EndpointUrl,
|
||||
/// Discord failed to correctly respond to IP discovery.
|
||||
IllegalDiscoveryResponse,
|
||||
/// Could not parse Discord's view of our IP.
|
||||
IllegalIp,
|
||||
/// Miscellaneous I/O error.
|
||||
Io(IoError),
|
||||
/// JSON (de)serialization error.
|
||||
Json(JsonError),
|
||||
/// Failed to message other background tasks after connection establishment.
|
||||
InterconnectFailure(Recipient),
|
||||
/// Error communicating with gateway server over WebSocket.
|
||||
Ws(WsError),
|
||||
/// Connection attempt timed out.
|
||||
TimedOut,
|
||||
}
|
||||
|
||||
impl From<CryptoError> for Error {
|
||||
fn from(e: CryptoError) -> Self {
|
||||
Error::Crypto(e)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<IoError> for Error {
|
||||
fn from(e: IoError) -> Error {
|
||||
Error::Io(e)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<JsonError> for Error {
|
||||
fn from(e: JsonError) -> Error {
|
||||
Error::Json(e)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SendError<WsMessage>> for Error {
|
||||
fn from(_e: SendError<WsMessage>) -> Error {
|
||||
Error::InterconnectFailure(Recipient::AuxNetwork)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SendError<EventMessage>> for Error {
|
||||
fn from(_e: SendError<EventMessage>) -> Error {
|
||||
Error::InterconnectFailure(Recipient::Event)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SendError<MixerMessage>> for Error {
|
||||
fn from(_e: SendError<MixerMessage>) -> Error {
|
||||
Error::InterconnectFailure(Recipient::Mixer)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<WsError> for Error {
|
||||
fn from(e: WsError) -> Error {
|
||||
Error::Ws(e)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Elapsed> for Error {
|
||||
fn from(_e: Elapsed) -> Error {
|
||||
Error::TimedOut
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Error {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "failed to connect to Discord RTP server: ")?;
|
||||
match self {
|
||||
Self::AttemptDiscarded => write!(f, "connection attempt was aborted/discarded"),
|
||||
Self::Crypto(e) => e.fmt(f),
|
||||
Self::CryptoInvalidLength => write!(f, "server supplied key of wrong length"),
|
||||
Self::CryptoModeInvalid => write!(f, "server changed negotiated encryption mode"),
|
||||
Self::CryptoModeUnavailable => write!(f, "server did not offer chosen encryption mode"),
|
||||
Self::DaveCreateKeyPackageError(e) => e.fmt(f),
|
||||
Self::DaveInitializationError(e) => e.fmt(f),
|
||||
Self::EndpointUrl => write!(f, "endpoint URL received from gateway was invalid"),
|
||||
Self::IllegalDiscoveryResponse => {
|
||||
write!(f, "IP discovery/NAT punching response was invalid")
|
||||
},
|
||||
Self::IllegalIp => write!(f, "IP discovery/NAT punching response had bad IP value"),
|
||||
Self::Io(e) => e.fmt(f),
|
||||
Self::Json(e) => e.fmt(f),
|
||||
Self::InterconnectFailure(e) => write!(f, "failed to contact other task ({e:?})"),
|
||||
Self::Ws(e) => write!(f, "websocket issue ({e:?})."),
|
||||
Self::TimedOut => write!(f, "connection attempt timed out"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl StdError for Error {
|
||||
fn source(&self) -> Option<&(dyn StdError + 'static)> {
|
||||
match self {
|
||||
Error::AttemptDiscarded
|
||||
| Error::Crypto(_)
|
||||
| Error::CryptoInvalidLength
|
||||
| Error::CryptoModeInvalid
|
||||
| Error::CryptoModeUnavailable
|
||||
| Error::DaveCreateKeyPackageError(_)
|
||||
| Error::DaveInitializationError(_)
|
||||
| Error::EndpointUrl
|
||||
| Error::IllegalDiscoveryResponse
|
||||
| Error::IllegalIp
|
||||
| Error::InterconnectFailure(_)
|
||||
| Error::Ws(_)
|
||||
| Error::TimedOut => None,
|
||||
Error::Io(e) => e.source(),
|
||||
Error::Json(e) => e.source(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Convenience type for Discord voice/driver connection error handling.
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
413
app/vendor/songbird/src/driver/connection/mod.rs
vendored
413
app/vendor/songbird/src/driver/connection/mod.rs
vendored
@@ -1,413 +0,0 @@
|
||||
pub mod error;
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
use super::tasks::udp_rx;
|
||||
use super::{
|
||||
crypto::Cipher,
|
||||
tasks::{
|
||||
message::*,
|
||||
ws::{self as ws_task, AuxNetwork},
|
||||
},
|
||||
Config,
|
||||
CryptoMode,
|
||||
};
|
||||
use crate::{
|
||||
constants::*,
|
||||
model::{
|
||||
payload::{Identify, Resume, SelectProtocol},
|
||||
Event as GatewayEvent,
|
||||
ProtocolData,
|
||||
},
|
||||
ws::WsStream,
|
||||
ConnectionInfo,
|
||||
};
|
||||
use discortp::discord::{IpDiscoveryPacket, IpDiscoveryType, MutableIpDiscoveryPacket};
|
||||
use error::{Error, Result};
|
||||
use flume::Sender;
|
||||
use serenity_voice_model::payload::DaveMlsKeyPackage;
|
||||
use socket2::Socket;
|
||||
use std::sync::{atomic::AtomicU16, Arc, RwLock};
|
||||
use std::{net::IpAddr, num::NonZeroU16, str::FromStr};
|
||||
use tokio::{net::UdpSocket, spawn, time::timeout};
|
||||
use tracing::{debug, info, instrument};
|
||||
use url::Url;
|
||||
|
||||
pub(crate) struct Connection {
|
||||
pub(crate) info: ConnectionInfo,
|
||||
pub(crate) ssrc: u32,
|
||||
pub(crate) ws: Sender<WsMessage>,
|
||||
}
|
||||
|
||||
impl Connection {
|
||||
pub(crate) async fn new(
|
||||
info: ConnectionInfo,
|
||||
interconnect: &Interconnect,
|
||||
config: &Config,
|
||||
idx: usize,
|
||||
) -> Result<Connection> {
|
||||
if let Some(t) = config.driver_timeout {
|
||||
timeout(
|
||||
t.into(),
|
||||
Connection::new_inner(info, interconnect, config, idx),
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
Connection::new_inner(info, interconnect, config, idx).await
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn new_inner(
|
||||
info: ConnectionInfo,
|
||||
interconnect: &Interconnect,
|
||||
config: &Config,
|
||||
idx: usize,
|
||||
) -> Result<Connection> {
|
||||
let url = generate_url(&info.endpoint)?;
|
||||
|
||||
let mut client = WsStream::connect(url).await?;
|
||||
let (ws_msg_tx, ws_msg_rx) = flume::unbounded();
|
||||
|
||||
let mut hello = None;
|
||||
let mut ready = None;
|
||||
|
||||
client
|
||||
.send_json(&GatewayEvent::from(Identify {
|
||||
server_id: info.guild_id.into(),
|
||||
session_id: info.session_id.clone(),
|
||||
token: info.token.clone(),
|
||||
user_id: info.user_id.into(),
|
||||
max_dave_protocol_version: Some(davey::DAVE_PROTOCOL_VERSION),
|
||||
}))
|
||||
.await?;
|
||||
|
||||
loop {
|
||||
let Some(value) = client.recv_event().await? else {
|
||||
continue;
|
||||
};
|
||||
|
||||
match value {
|
||||
GatewayEvent::Ready(r) => {
|
||||
ready = Some(r);
|
||||
if hello.is_some() {
|
||||
break;
|
||||
}
|
||||
},
|
||||
GatewayEvent::Hello(h) => {
|
||||
hello = Some(h);
|
||||
if ready.is_some() {
|
||||
break;
|
||||
}
|
||||
},
|
||||
other => {
|
||||
// Discord hold back per-user connection state until after this handshake.
|
||||
// There's no guarantee that will remain the case, so buffer it like all
|
||||
// subsequent steps where we know they *do* send these packets.
|
||||
debug!("Expected ready/hello; got: {:?}", other);
|
||||
ws_msg_tx.send(WsMessage::Deliver(other))?;
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
let hello =
|
||||
hello.expect("Hello packet expected in connection initialisation, but not found.");
|
||||
let ready =
|
||||
ready.expect("Ready packet expected in connection initialisation, but not found.");
|
||||
|
||||
let chosen_crypto = CryptoMode::negotiate(&ready.modes, Some(config.crypto_mode))?;
|
||||
|
||||
info!(
|
||||
"Crypto scheme negotiation -- wanted {:?}. Chose {:?} from modes {:?}.",
|
||||
config.crypto_mode, chosen_crypto, ready.modes
|
||||
);
|
||||
|
||||
let udp = UdpSocket::bind("0.0.0.0:0").await?;
|
||||
|
||||
// Optimisation for non-receive case: set rx buffer size to zero.
|
||||
let udp = if cfg!(feature = "receive") {
|
||||
udp
|
||||
} else {
|
||||
let socket = Socket::from(udp.into_std()?);
|
||||
|
||||
// Some operating systems do not allow setting the recv buffer to 0.
|
||||
#[cfg(any(target_os = "linux", target_os = "windows"))]
|
||||
socket.set_recv_buffer_size(0)?;
|
||||
|
||||
UdpSocket::from_std(socket.into())?
|
||||
};
|
||||
|
||||
udp.connect((ready.ip, ready.port)).await?;
|
||||
|
||||
// Follow Discord's IP Discovery procedures, in case NAT tunnelling is needed.
|
||||
let mut bytes = [0; IpDiscoveryPacket::const_packet_size()];
|
||||
{
|
||||
let mut view = MutableIpDiscoveryPacket::new(&mut bytes[..]).expect(
|
||||
"Too few bytes in 'bytes' for IPDiscovery packet.\
|
||||
(Blame: IpDiscoveryPacket::const_packet_size()?)",
|
||||
);
|
||||
view.set_pkt_type(IpDiscoveryType::Request);
|
||||
view.set_length(70);
|
||||
view.set_ssrc(ready.ssrc);
|
||||
}
|
||||
|
||||
udp.send(&bytes).await?;
|
||||
|
||||
let (len, _addr) = udp.recv_from(&mut bytes).await?;
|
||||
{
|
||||
let view =
|
||||
IpDiscoveryPacket::new(&bytes[..len]).ok_or(Error::IllegalDiscoveryResponse)?;
|
||||
|
||||
if view.get_pkt_type() != IpDiscoveryType::Response {
|
||||
return Err(Error::IllegalDiscoveryResponse);
|
||||
}
|
||||
|
||||
// We could do something clever like binary search,
|
||||
// but possibility of UDP spoofing precludes us from
|
||||
// making the assumption we can find a "left edge" of '\0's.
|
||||
let nul_byte_index = view
|
||||
.get_address_raw()
|
||||
.iter()
|
||||
.position(|&b| b == 0)
|
||||
.ok_or(Error::IllegalIp)?;
|
||||
|
||||
let address_str = std::str::from_utf8(&view.get_address_raw()[..nul_byte_index])
|
||||
.map_err(|_| Error::IllegalIp)?;
|
||||
|
||||
let address = IpAddr::from_str(address_str).map_err(|_| Error::IllegalIp)?;
|
||||
|
||||
client
|
||||
.send_json(&GatewayEvent::from(SelectProtocol {
|
||||
protocol: "udp".into(),
|
||||
data: ProtocolData {
|
||||
address,
|
||||
mode: chosen_crypto.to_request_str().into(),
|
||||
port: view.get_port(),
|
||||
},
|
||||
}))
|
||||
.await?;
|
||||
}
|
||||
|
||||
let (cipher, dave_session, dave_protocol_version) =
|
||||
init_cipher(&mut client, &info, chosen_crypto, &ws_msg_tx).await?;
|
||||
let dave_session = Arc::new(RwLock::new(dave_session));
|
||||
let dave_protocol_version = Arc::new(dave_protocol_version);
|
||||
|
||||
info!("Connected to: {}", info.endpoint);
|
||||
|
||||
info!("WS heartbeat duration {}ms.", hello.heartbeat_interval);
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
let (udp_receiver_msg_tx, udp_receiver_msg_rx) = flume::unbounded();
|
||||
|
||||
// NOTE: This causes the UDP Socket on "receive" to be non-blocking,
|
||||
// and the standard to be blocking. A UDP send should only WouldBlock if
|
||||
// you're sending more data than the OS can handle (not likely, and
|
||||
// at that point you should scale horizontally).
|
||||
//
|
||||
// If this is a problem for anyone, we can make non-blocking sends
|
||||
// queue up a delayed send up to a limit.
|
||||
#[cfg(feature = "receive")]
|
||||
let (udp_rx, udp_tx) = {
|
||||
let udp_tx = udp.into_std()?;
|
||||
let udp_rx = UdpSocket::from_std(udp_tx.try_clone()?)?;
|
||||
(udp_rx, udp_tx)
|
||||
};
|
||||
#[cfg(not(feature = "receive"))]
|
||||
let udp_tx = udp.into_std()?;
|
||||
|
||||
let ssrc = ready.ssrc;
|
||||
|
||||
let mix_conn = MixerConnection {
|
||||
#[cfg(feature = "receive")]
|
||||
cipher: cipher.clone(),
|
||||
#[cfg(not(feature = "receive"))]
|
||||
cipher,
|
||||
dave_session: dave_session.clone(),
|
||||
dave_protocol_version: dave_protocol_version.clone(),
|
||||
crypto_state: chosen_crypto.into(),
|
||||
#[cfg(feature = "receive")]
|
||||
udp_rx: udp_receiver_msg_tx,
|
||||
udp_tx,
|
||||
};
|
||||
|
||||
interconnect
|
||||
.mixer
|
||||
.send(MixerMessage::Ws(Some(ws_msg_tx.clone())))?;
|
||||
|
||||
interconnect
|
||||
.mixer
|
||||
.send(MixerMessage::SetConn(mix_conn, ready.ssrc))?;
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
let ssrc_tracker = Arc::new(SsrcTracker::default());
|
||||
|
||||
let ws_state = AuxNetwork::new(
|
||||
ws_msg_rx,
|
||||
client,
|
||||
ssrc,
|
||||
hello.heartbeat_interval,
|
||||
idx,
|
||||
info.clone(),
|
||||
#[cfg(not(feature = "receive"))]
|
||||
dave_session,
|
||||
#[cfg(not(feature = "receive"))]
|
||||
dave_protocol_version,
|
||||
#[cfg(feature = "receive")]
|
||||
dave_session.clone(),
|
||||
#[cfg(feature = "receive")]
|
||||
dave_protocol_version.clone(),
|
||||
#[cfg(feature = "receive")]
|
||||
ssrc_tracker.clone(),
|
||||
);
|
||||
|
||||
spawn(ws_task::runner(interconnect.clone(), ws_state));
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
spawn(udp_rx::runner(
|
||||
interconnect.clone(),
|
||||
udp_receiver_msg_rx,
|
||||
cipher,
|
||||
chosen_crypto,
|
||||
config.clone(),
|
||||
udp_rx,
|
||||
ssrc_tracker,
|
||||
dave_session,
|
||||
dave_protocol_version,
|
||||
));
|
||||
|
||||
Ok(Connection {
|
||||
info,
|
||||
ssrc,
|
||||
ws: ws_msg_tx,
|
||||
})
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub async fn reconnect(&mut self, config: &Config) -> Result<()> {
|
||||
if let Some(t) = config.driver_timeout {
|
||||
timeout(t.into(), self.reconnect_inner()).await?
|
||||
} else {
|
||||
self.reconnect_inner().await
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub async fn reconnect_inner(&mut self) -> Result<()> {
|
||||
let url = generate_url(&self.info.endpoint)?;
|
||||
|
||||
// Thread may have died, we want to send to prompt a clean exit
|
||||
// (if at all possible) and then proceed as normal.
|
||||
let mut client = WsStream::connect(url).await?;
|
||||
|
||||
client
|
||||
.send_json(&GatewayEvent::from(Resume {
|
||||
server_id: self.info.guild_id.into(),
|
||||
session_id: self.info.session_id.clone(),
|
||||
token: self.info.token.clone(),
|
||||
}))
|
||||
.await?;
|
||||
|
||||
let mut hello = None;
|
||||
let mut resumed = None;
|
||||
|
||||
loop {
|
||||
let Some(value) = client.recv_event().await? else {
|
||||
continue;
|
||||
};
|
||||
|
||||
match value {
|
||||
GatewayEvent::Resumed => {
|
||||
resumed = Some(());
|
||||
if hello.is_some() {
|
||||
break;
|
||||
}
|
||||
},
|
||||
GatewayEvent::Hello(h) => {
|
||||
hello = Some(h);
|
||||
if resumed.is_some() {
|
||||
break;
|
||||
}
|
||||
},
|
||||
other => {
|
||||
self.ws.send(WsMessage::Deliver(other))?;
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
let hello =
|
||||
hello.expect("Hello packet expected in connection initialisation, but not found.");
|
||||
|
||||
self.ws
|
||||
.send(WsMessage::SetKeepalive(hello.heartbeat_interval))?;
|
||||
self.ws.send(WsMessage::Ws(Box::new(client)))?;
|
||||
|
||||
info!("Reconnected to: {}", &self.info.endpoint);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Connection {
|
||||
fn drop(&mut self) {
|
||||
info!("Disconnected");
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_url(endpoint: &str) -> Result<Url> {
|
||||
Url::parse(&format!("wss://{endpoint}/?v={VOICE_GATEWAY_VERSION}")).or(Err(Error::EndpointUrl))
|
||||
}
|
||||
|
||||
#[inline]
|
||||
async fn init_cipher(
|
||||
client: &mut WsStream,
|
||||
info: &ConnectionInfo,
|
||||
mode: CryptoMode,
|
||||
tx: &Sender<WsMessage>,
|
||||
) -> Result<(Cipher, Option<davey::DaveSession>, AtomicU16)> {
|
||||
loop {
|
||||
let Some(value) = client.recv_event().await? else {
|
||||
continue;
|
||||
};
|
||||
|
||||
match value {
|
||||
GatewayEvent::SessionDescription(desc) => {
|
||||
if desc.mode != mode.to_request_str() {
|
||||
return Err(Error::CryptoModeInvalid);
|
||||
}
|
||||
|
||||
let dave_session =
|
||||
if let Some(version) = NonZeroU16::new(desc.dave_protocol_version) {
|
||||
let mut session = davey::DaveSession::new(
|
||||
version,
|
||||
info.user_id.0.into(),
|
||||
info.channel_id.0.into(),
|
||||
None,
|
||||
)
|
||||
.map_err(Error::DaveInitializationError)?;
|
||||
|
||||
client
|
||||
.send_binary(&GatewayEvent::DaveMlsKeyPackage(DaveMlsKeyPackage {
|
||||
key_package: session
|
||||
.create_key_package()
|
||||
.map_err(Error::DaveCreateKeyPackageError)?,
|
||||
}))
|
||||
.await?;
|
||||
|
||||
Some(session)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
return Ok((
|
||||
mode.cipher_from_key(&desc.secret_key)
|
||||
.map_err(|_| Error::CryptoInvalidLength)?,
|
||||
dave_session,
|
||||
AtomicU16::new(desc.dave_protocol_version),
|
||||
));
|
||||
},
|
||||
other => {
|
||||
// Discord can and will send user-specific payload packets during this time
|
||||
// which are needed to map SSRCs to `UserId`s.
|
||||
tx.send(WsMessage::Deliver(other))?;
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
578
app/vendor/songbird/src/driver/crypto.rs
vendored
578
app/vendor/songbird/src/driver/crypto.rs
vendored
@@ -1,578 +0,0 @@
|
||||
//! Encryption schemes supported by Discord's secure RTP negotiation.
|
||||
#[cfg(any(feature = "receive", test))]
|
||||
use super::tasks::error::Error as InternalError;
|
||||
use aead::AeadCore;
|
||||
use aes_gcm::{AeadInPlace, Aes256Gcm, Error as CryptoError};
|
||||
use byteorder::{NetworkEndian, WriteBytesExt};
|
||||
use chacha20poly1305::XChaCha20Poly1305;
|
||||
use crypto_common::{InvalidLength, KeyInit};
|
||||
#[cfg(feature = "receive")]
|
||||
use discortp::rtcp::MutableRtcpPacket;
|
||||
use discortp::MutablePacket;
|
||||
#[cfg(any(feature = "receive", test))]
|
||||
use discortp::{
|
||||
rtp::{MutableRtpPacket, RtpExtensionPacket},
|
||||
Packet,
|
||||
};
|
||||
use std::{num::Wrapping, str::FromStr};
|
||||
use typenum::Unsigned;
|
||||
|
||||
use crate::error::ConnectionError;
|
||||
|
||||
/// Encryption schemes supportd by Discord.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq, Default, Hash)]
|
||||
#[non_exhaustive]
|
||||
pub enum CryptoMode {
|
||||
#[default]
|
||||
/// Discord's currently preferred non-E2EE encryption scheme.
|
||||
///
|
||||
/// Packets are encrypted and decrypted using the `AES256GCM` encryption scheme.
|
||||
/// An additional random 4B suffix is used as the source of nonce bytes for the packet.
|
||||
/// This nonce value increments by `1` with each packet.
|
||||
///
|
||||
/// Encrypted content begins *after* the RTP header and extensions, following the SRTP
|
||||
/// specification.
|
||||
///
|
||||
/// Nonce width of 4B (32b), at an extra 4B per packet (~0.2 kB/s).
|
||||
Aes256Gcm,
|
||||
/// A fallback non-E2EE encryption scheme.
|
||||
///
|
||||
/// Packets are encrypted and decrypted using the `XChaCha20Poly1305` encryption scheme.
|
||||
/// An additional random 4B suffix is used as the source of nonce bytes for the packet.
|
||||
/// This nonce value increments by `1` with each packet.
|
||||
///
|
||||
/// Encrypted content begins *after* the RTP header and extensions, following the SRTP
|
||||
/// specification.
|
||||
///
|
||||
/// Nonce width of 4B (32b), at an extra 4B per packet (~0.2 kB/s).
|
||||
XChaCha20Poly1305,
|
||||
}
|
||||
|
||||
impl From<CryptoState> for CryptoMode {
|
||||
fn from(val: CryptoState) -> Self {
|
||||
match val {
|
||||
CryptoState::Aes256Gcm(_) => Self::Aes256Gcm,
|
||||
CryptoState::XChaCha20Poly1305(_) => Self::XChaCha20Poly1305,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The input string could not be parsed as an encryption scheme supported by songbird.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
|
||||
pub struct UnrecognisedCryptoMode;
|
||||
|
||||
impl FromStr for CryptoMode {
|
||||
type Err = UnrecognisedCryptoMode;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s {
|
||||
"aead_aes256_gcm_rtpsize" => Ok(Self::Aes256Gcm),
|
||||
"aead_xchacha20_poly1305_rtpsize" => Ok(Self::XChaCha20Poly1305),
|
||||
_ => Err(UnrecognisedCryptoMode),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CryptoMode {
|
||||
/// Returns the underlying crypto algorithm used by a given [`CryptoMode`].
|
||||
#[must_use]
|
||||
pub(crate) const fn algorithm(self) -> EncryptionAlgorithm {
|
||||
match self {
|
||||
CryptoMode::Aes256Gcm => EncryptionAlgorithm::Aes256Gcm,
|
||||
CryptoMode::XChaCha20Poly1305 => EncryptionAlgorithm::XChaCha20Poly1305,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns an encryption cipher based on the supplied key.
|
||||
///
|
||||
/// Creation fails if the key is the incorrect length for the target cipher.
|
||||
pub(crate) fn cipher_from_key(self, key: &[u8]) -> Result<Cipher, InvalidLength> {
|
||||
match self.algorithm() {
|
||||
EncryptionAlgorithm::Aes256Gcm => Aes256Gcm::new_from_slice(key)
|
||||
.map(Box::new)
|
||||
.map(Cipher::Aes256Gcm),
|
||||
EncryptionAlgorithm::XChaCha20Poly1305 =>
|
||||
XChaCha20Poly1305::new_from_slice(key).map(Cipher::XChaCha20Poly1305),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a local priority score for a given [`CryptoMode`].
|
||||
///
|
||||
/// Higher values are preferred.
|
||||
#[must_use]
|
||||
pub(crate) fn priority(self) -> u64 {
|
||||
match self {
|
||||
CryptoMode::Aes256Gcm => 1,
|
||||
CryptoMode::XChaCha20Poly1305 => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the best available crypto mode, given the `modes` offered by the Discord voice server.
|
||||
///
|
||||
/// If `preferred` is set and the mode exists in the server's supported algorithms, then that
|
||||
/// mode will be chosen. Otherwise we select the highest-scoring option which is mutually understood.
|
||||
pub(crate) fn negotiate<It, T>(
|
||||
modes: It,
|
||||
preferred: Option<Self>,
|
||||
) -> Result<Self, ConnectionError>
|
||||
where
|
||||
T: AsRef<str>,
|
||||
It: IntoIterator<Item = T>,
|
||||
{
|
||||
let mut best = None;
|
||||
for el in modes {
|
||||
let Ok(el) = CryptoMode::from_str(el.as_ref()) else {
|
||||
// Unsupported mode. Ignore.
|
||||
continue;
|
||||
};
|
||||
|
||||
let mut el_priority = el.priority();
|
||||
if let Some(preferred) = preferred {
|
||||
if el == preferred {
|
||||
el_priority = u64::MAX;
|
||||
}
|
||||
}
|
||||
|
||||
let accept = match best {
|
||||
None => true,
|
||||
Some((_, score)) if el_priority > score => true,
|
||||
_ => false,
|
||||
};
|
||||
|
||||
if accept {
|
||||
best = Some((el, el_priority));
|
||||
}
|
||||
}
|
||||
|
||||
best.map(|(v, _)| v)
|
||||
.ok_or(ConnectionError::CryptoModeUnavailable)
|
||||
}
|
||||
|
||||
/// Returns the name of a mode as it will appear during negotiation.
|
||||
#[must_use]
|
||||
pub const fn to_request_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Aes256Gcm => "aead_aes256_gcm_rtpsize",
|
||||
Self::XChaCha20Poly1305 => "aead_xchacha20_poly1305_rtpsize",
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the nonce length in bytes required by algorithm.
|
||||
#[must_use]
|
||||
pub const fn algorithm_nonce_size(self) -> usize {
|
||||
use typenum::Unsigned as _;
|
||||
match self {
|
||||
Self::XChaCha20Poly1305 => <XChaCha20Poly1305 as AeadCore>::NonceSize::USIZE, // => 24
|
||||
Self::Aes256Gcm => <Aes256Gcm as AeadCore>::NonceSize::USIZE, // => 12
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of bytes each nonce is stored as within
|
||||
/// a packet.
|
||||
#[must_use]
|
||||
pub const fn nonce_size(self) -> usize {
|
||||
match self {
|
||||
Self::Aes256Gcm | Self::XChaCha20Poly1305 => 4,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of bytes occupied by the encryption scheme
|
||||
/// which fall before the payload.
|
||||
///
|
||||
/// Method name duplicated until v0.5, to prevent breaking change.
|
||||
#[must_use]
|
||||
pub(crate) const fn payload_prefix_len(self) -> usize {
|
||||
match self {
|
||||
CryptoMode::Aes256Gcm | CryptoMode::XChaCha20Poly1305 => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the tag length in bytes.
|
||||
#[must_use]
|
||||
pub(crate) const fn encryption_tag_len(self) -> usize {
|
||||
self.algorithm().encryption_tag_len()
|
||||
}
|
||||
|
||||
/// Returns the number of bytes occupied by the encryption scheme
|
||||
/// which fall after the payload.
|
||||
#[must_use]
|
||||
pub const fn payload_suffix_len(self) -> usize {
|
||||
self.nonce_size() + self.encryption_tag_len()
|
||||
}
|
||||
|
||||
/// Returns the number of bytes occupied by an encryption scheme's tag which
|
||||
/// fall *after* the payload.
|
||||
#[must_use]
|
||||
pub const fn tag_suffix_len(self) -> usize {
|
||||
self.encryption_tag_len()
|
||||
}
|
||||
|
||||
/// Calculates the number of additional bytes required compared
|
||||
/// to an unencrypted payload.
|
||||
#[must_use]
|
||||
pub const fn payload_overhead(self) -> usize {
|
||||
self.payload_prefix_len() + self.payload_suffix_len()
|
||||
}
|
||||
|
||||
/// Extracts the byte slice in a packet used as the nonce, and the remaining mutable
|
||||
/// portion of the packet.
|
||||
fn nonce_slice<'a>(
|
||||
self,
|
||||
_header: &'a [u8],
|
||||
body: &'a mut [u8],
|
||||
) -> Result<(&'a [u8], &'a mut [u8]), CryptoError> {
|
||||
match self {
|
||||
Self::Aes256Gcm | Self::XChaCha20Poly1305 => {
|
||||
let len = body.len();
|
||||
if len < self.payload_suffix_len() {
|
||||
Err(CryptoError)
|
||||
} else {
|
||||
let (body_left, nonce_loc) = body.split_at_mut(len - self.nonce_size());
|
||||
Ok((nonce_loc, body_left))
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// State used in nonce generation for the encryption variants in [`CryptoMode`].
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
#[non_exhaustive]
|
||||
pub enum CryptoState {
|
||||
/// An additional random 4B suffix is used as the source of nonce bytes for the packet.
|
||||
/// This nonce value increments by `1` with each packet.
|
||||
///
|
||||
/// The last used nonce is stored.
|
||||
Aes256Gcm(Wrapping<u32>),
|
||||
/// An additional random 4B suffix is used as the source of nonce bytes for the packet.
|
||||
/// This nonce value increments by `1` with each packet.
|
||||
///
|
||||
/// The last used nonce is stored.
|
||||
XChaCha20Poly1305(Wrapping<u32>),
|
||||
}
|
||||
|
||||
impl From<CryptoMode> for CryptoState {
|
||||
fn from(val: CryptoMode) -> Self {
|
||||
match val {
|
||||
CryptoMode::Aes256Gcm => CryptoState::Aes256Gcm(Wrapping(rand::random::<u32>())),
|
||||
CryptoMode::XChaCha20Poly1305 =>
|
||||
CryptoState::XChaCha20Poly1305(Wrapping(rand::random::<u32>())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CryptoState {
|
||||
/// Writes packet nonce into the body, if required, returning the new length.
|
||||
pub fn write_packet_nonce(
|
||||
&mut self,
|
||||
packet: &mut impl MutablePacket,
|
||||
payload_end: usize,
|
||||
) -> usize {
|
||||
let mode = self.kind();
|
||||
let endpoint = payload_end + mode.payload_suffix_len();
|
||||
let startpoint = endpoint - mode.nonce_size();
|
||||
|
||||
match self {
|
||||
Self::Aes256Gcm(ref mut i) | Self::XChaCha20Poly1305(ref mut i) => {
|
||||
(&mut packet.payload_mut()[startpoint..endpoint])
|
||||
.write_u32::<NetworkEndian>(i.0)
|
||||
.expect(
|
||||
"Nonce size is guaranteed to be sufficient to write u32 for lite tagging.",
|
||||
);
|
||||
*i += Wrapping(1);
|
||||
},
|
||||
}
|
||||
|
||||
endpoint
|
||||
}
|
||||
|
||||
/// Returns the underlying (stateless) type of the active crypto mode.
|
||||
#[must_use]
|
||||
pub fn kind(self) -> CryptoMode {
|
||||
CryptoMode::from(self)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
|
||||
pub(crate) enum EncryptionAlgorithm {
|
||||
Aes256Gcm,
|
||||
XChaCha20Poly1305,
|
||||
}
|
||||
|
||||
impl EncryptionAlgorithm {
|
||||
#[must_use]
|
||||
pub(crate) const fn encryption_tag_len(self) -> usize {
|
||||
match self {
|
||||
Self::Aes256Gcm => <Aes256Gcm as AeadCore>::TagSize::USIZE, // 16
|
||||
Self::XChaCha20Poly1305 => <XChaCha20Poly1305 as AeadCore>::TagSize::USIZE, // 16
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&Cipher> for EncryptionAlgorithm {
|
||||
fn from(value: &Cipher) -> Self {
|
||||
match value {
|
||||
Cipher::XChaCha20Poly1305(_) => EncryptionAlgorithm::XChaCha20Poly1305,
|
||||
Cipher::Aes256Gcm(_) => EncryptionAlgorithm::Aes256Gcm,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub enum Cipher {
|
||||
XChaCha20Poly1305(XChaCha20Poly1305),
|
||||
Aes256Gcm(Box<Aes256Gcm>),
|
||||
}
|
||||
|
||||
impl Cipher {
|
||||
#[must_use]
|
||||
pub(crate) fn mode(&self) -> CryptoMode {
|
||||
match self {
|
||||
Cipher::XChaCha20Poly1305(_) => CryptoMode::XChaCha20Poly1305,
|
||||
Cipher::Aes256Gcm(_) => CryptoMode::Aes256Gcm,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub(crate) fn encryption_tag_len(&self) -> usize {
|
||||
EncryptionAlgorithm::from(self).encryption_tag_len()
|
||||
}
|
||||
|
||||
/// Encrypts a Discord RT(C)P packet using the given key.
|
||||
///
|
||||
/// Use of this requires that the input packet has had a nonce generated in the correct location,
|
||||
/// and `payload_len` specifies the number of bytes after the header including this nonce.
|
||||
#[inline]
|
||||
pub fn encrypt_pkt_in_place(
|
||||
&self,
|
||||
packet: &mut impl MutablePacket,
|
||||
payload_len: usize,
|
||||
) -> Result<(), CryptoError> {
|
||||
let mode = self.mode();
|
||||
let header_len = packet.packet().len() - packet.payload().len();
|
||||
|
||||
let (header, body) = packet.packet_mut().split_at_mut(header_len);
|
||||
let (slice_to_use, body_remaining) = mode.nonce_slice(header, &mut body[..payload_len])?;
|
||||
|
||||
let tag_size = self.encryption_tag_len();
|
||||
|
||||
// body_remaining is now correctly truncated to exclude the nonce by this point.
|
||||
// the true_payload to encrypt is within the buf[prefix:-suffix].
|
||||
let (_, body_remaining) = body_remaining.split_at_mut(mode.payload_prefix_len());
|
||||
let (body, post_payload) =
|
||||
body_remaining.split_at_mut(body_remaining.len() - mode.tag_suffix_len());
|
||||
|
||||
// All these Nonce types are distinct at the type level
|
||||
// (96b for AES, 192b for XChaCha).
|
||||
match self {
|
||||
// The below variants follow part of the SRTP spec (RFC3711, sec 3.1)
|
||||
// by requiring that we include the cleartext header portion as
|
||||
// authenticated data.
|
||||
Self::Aes256Gcm(aes_gcm) => {
|
||||
let mut nonce = aes_gcm::Nonce::default();
|
||||
nonce[..mode.nonce_size()].copy_from_slice(slice_to_use);
|
||||
|
||||
let tag = aes_gcm.encrypt_in_place_detached(&nonce, header, body)?;
|
||||
post_payload[..tag_size].copy_from_slice(&tag[..]);
|
||||
},
|
||||
Self::XChaCha20Poly1305(cha_cha_poly1305) => {
|
||||
let mut nonce = chacha20poly1305::XNonce::default();
|
||||
nonce[..mode.nonce_size()].copy_from_slice(slice_to_use);
|
||||
|
||||
let tag = cha_cha_poly1305.encrypt_in_place_detached(&nonce, header, body)?;
|
||||
post_payload[..tag_size].copy_from_slice(&tag[..]);
|
||||
},
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "receive", test))]
|
||||
pub(crate) fn decrypt_rtp_in_place(
|
||||
&self,
|
||||
packet: &mut MutableRtpPacket<'_>,
|
||||
) -> Result<(usize, usize), InternalError> {
|
||||
// An exciting difference from the SRTP spec: Discord begins encryption
|
||||
// after the RTP extension *header*, encrypting the extensions themselves,
|
||||
// whereas the spec leaves all extensions in the clear.
|
||||
// This header is described as the 'extension preamble'.
|
||||
let has_extension = packet.get_extension() != 0;
|
||||
|
||||
let plain_bytes = if has_extension {
|
||||
// CSRCs and extension bytes will be in the plaintext segment.
|
||||
// We will need these demarcated to select the right bytes to
|
||||
// decrypt, and to use as auth data.
|
||||
RtpExtensionPacket::minimum_packet_size()
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
let (_, end) = self.decrypt_pkt_in_place(packet, plain_bytes)?;
|
||||
|
||||
// Update the start estimate to account for bytes occupied by extension headers.
|
||||
let payload_offset = if has_extension {
|
||||
let payload = packet.payload();
|
||||
let extension =
|
||||
RtpExtensionPacket::new(payload).ok_or(InternalError::IllegalVoicePacket)?;
|
||||
extension.packet().len() - extension.payload().len()
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
Ok((payload_offset, end))
|
||||
}
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
pub(crate) fn decrypt_rtcp_in_place(
|
||||
&self,
|
||||
packet: &mut MutableRtcpPacket<'_>,
|
||||
) -> Result<(usize, usize), InternalError> {
|
||||
// RTCP/SRTCP have identical handling -- no var-length elements
|
||||
// are included as part of the plaintext.
|
||||
self.decrypt_pkt_in_place(packet, 0)
|
||||
}
|
||||
|
||||
/// Decrypts an arbitrary packet using the given key.
|
||||
///
|
||||
/// If successful, this returns the number of bytes to be ignored from the
|
||||
/// start and end of the packet payload.
|
||||
#[inline]
|
||||
#[cfg(any(feature = "receive", test))]
|
||||
pub(crate) fn decrypt_pkt_in_place(
|
||||
&self,
|
||||
packet: &mut impl MutablePacket,
|
||||
n_plaintext_body_bytes: usize,
|
||||
) -> Result<(usize, usize), InternalError> {
|
||||
let mode = self.mode();
|
||||
let header_len = packet.packet().len() - packet.payload().len();
|
||||
let plaintext_end = header_len + n_plaintext_body_bytes;
|
||||
|
||||
let (plaintext, ciphertext) = packet
|
||||
.packet_mut()
|
||||
.split_at_mut_checked(plaintext_end)
|
||||
.ok_or(CryptoError)?;
|
||||
let (slice_to_use, body_remaining) = mode.nonce_slice(plaintext, ciphertext)?;
|
||||
|
||||
let (pre_payload, body_remaining) = body_remaining
|
||||
.split_at_mut_checked(mode.payload_prefix_len())
|
||||
.ok_or(CryptoError)?;
|
||||
|
||||
let suffix_split_point = body_remaining
|
||||
.len()
|
||||
.checked_sub(mode.tag_suffix_len())
|
||||
.ok_or(CryptoError)?;
|
||||
|
||||
let (body, post_payload) = body_remaining
|
||||
.split_at_mut_checked(suffix_split_point)
|
||||
.ok_or(CryptoError)?;
|
||||
|
||||
let tag_size = self.encryption_tag_len();
|
||||
|
||||
match self {
|
||||
// The below variants follow part of the SRTP spec (RFC3711, sec 3.1)
|
||||
// by requiring that we include the cleartext header portion as
|
||||
// authenticated data.
|
||||
Self::Aes256Gcm(aes_gcm) => {
|
||||
let mut nonce = aes_gcm::Nonce::default();
|
||||
nonce[..mode.nonce_size()].copy_from_slice(slice_to_use);
|
||||
|
||||
let tag = aes_gcm::Tag::from_slice(&post_payload[..tag_size]);
|
||||
aes_gcm.decrypt_in_place_detached(&nonce, plaintext, body, tag)?;
|
||||
},
|
||||
Self::XChaCha20Poly1305(cha_cha_poly1305) => {
|
||||
let mut nonce = chacha20poly1305::XNonce::default();
|
||||
nonce[..mode.nonce_size()].copy_from_slice(slice_to_use);
|
||||
|
||||
let tag = chacha20poly1305::Tag::from_slice(&post_payload[..tag_size]);
|
||||
cha_cha_poly1305.decrypt_in_place_detached(&nonce, plaintext, body, tag)?;
|
||||
},
|
||||
}
|
||||
|
||||
Ok((
|
||||
plaintext_end + pre_payload.len(),
|
||||
post_payload.len() + slice_to_use.len(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
use discortp::rtp::MutableRtpPacket;
|
||||
|
||||
#[test]
|
||||
|
||||
fn small_packet_decrypts_error() {
|
||||
let mut buf = [0u8; MutableRtpPacket::minimum_packet_size()];
|
||||
let modes = [CryptoMode::Aes256Gcm, CryptoMode::XChaCha20Poly1305];
|
||||
let mut pkt = MutableRtpPacket::new(&mut buf[..]).unwrap();
|
||||
|
||||
for mode in modes {
|
||||
// Coincidentally, these are all 32B for now.
|
||||
let cipher = mode.cipher_from_key(&[1u8; 32]).unwrap();
|
||||
// AIM: should error, and not panic.
|
||||
assert!(cipher.decrypt_rtp_in_place(&mut pkt).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn symmetric_encrypt_decrypt_tag_after_data() {
|
||||
const TRUE_PAYLOAD: [u8; 8] = [1, 2, 3, 4, 5, 6, 7, 8];
|
||||
for mode in [CryptoMode::Aes256Gcm, CryptoMode::XChaCha20Poly1305] {
|
||||
let mut buf = vec![
|
||||
0u8;
|
||||
MutableRtpPacket::minimum_packet_size()
|
||||
+ TRUE_PAYLOAD.len()
|
||||
+ mode.nonce_size()
|
||||
+ mode.encryption_tag_len()
|
||||
];
|
||||
|
||||
buf.fill(0);
|
||||
let cipher = mode.cipher_from_key(&[7u8; 32]).unwrap();
|
||||
let mut pkt = MutableRtpPacket::new(&mut buf[..]).unwrap();
|
||||
let mut crypto_state = CryptoState::from(mode);
|
||||
let payload = pkt.payload_mut();
|
||||
payload[mode.payload_prefix_len()..TRUE_PAYLOAD.len()].copy_from_slice(&TRUE_PAYLOAD);
|
||||
|
||||
let final_payload_size = crypto_state.write_packet_nonce(&mut pkt, TRUE_PAYLOAD.len());
|
||||
|
||||
let enc_succ = cipher.encrypt_pkt_in_place(&mut pkt, final_payload_size);
|
||||
|
||||
assert!(enc_succ.is_ok());
|
||||
|
||||
let final_pkt_len = MutableRtpPacket::minimum_packet_size() + final_payload_size;
|
||||
let mut pkt = MutableRtpPacket::new(&mut buf[..final_pkt_len]).unwrap();
|
||||
|
||||
assert!(cipher.decrypt_rtp_in_place(&mut pkt).is_ok());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
fn negotiate_cryptomode() {
|
||||
// If we have no preference (or our preference is missing), choose the highest available in the set.
|
||||
let test_set =
|
||||
[CryptoMode::XChaCha20Poly1305, CryptoMode::Aes256Gcm].map(CryptoMode::to_request_str);
|
||||
assert_eq!(
|
||||
CryptoMode::negotiate(test_set, None).unwrap(),
|
||||
CryptoMode::Aes256Gcm
|
||||
);
|
||||
|
||||
let test_set_missing = [CryptoMode::XChaCha20Poly1305].map(CryptoMode::to_request_str);
|
||||
assert_eq!(
|
||||
CryptoMode::negotiate(test_set_missing, Some(CryptoMode::Aes256Gcm)).unwrap(),
|
||||
CryptoMode::XChaCha20Poly1305
|
||||
);
|
||||
|
||||
// Preference wins in spite of the defined `priority` value.
|
||||
assert_eq!(
|
||||
CryptoMode::negotiate(test_set, Some(CryptoMode::XChaCha20Poly1305)).unwrap(),
|
||||
CryptoMode::XChaCha20Poly1305
|
||||
);
|
||||
|
||||
// If there is no mutual intelligibility, return an error.
|
||||
let bad_modes = ["not_real", "des", "rc5"];
|
||||
assert!(CryptoMode::negotiate(bad_modes, None).is_err());
|
||||
assert!(CryptoMode::negotiate(bad_modes, Some(CryptoMode::Aes256Gcm)).is_err());
|
||||
}
|
||||
}
|
||||
125
app/vendor/songbird/src/driver/decode_mode.rs
vendored
125
app/vendor/songbird/src/driver/decode_mode.rs
vendored
@@ -1,125 +0,0 @@
|
||||
use opus2::Channels as OpusChannels;
|
||||
|
||||
/// Decode behaviour for received RTP packets within the driver.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
|
||||
#[non_exhaustive]
|
||||
pub enum DecodeMode {
|
||||
/// Packets received from Discord are handed over to events without any
|
||||
/// changes applied.
|
||||
///
|
||||
/// No CPU work involved.
|
||||
Pass,
|
||||
/// Decrypts the body of each received packet.
|
||||
///
|
||||
/// Small per-packet CPU use.
|
||||
Decrypt,
|
||||
/// Decrypts and decodes each received packet, correctly accounting for losses.
|
||||
///
|
||||
/// Larger per-packet CPU use.
|
||||
Decode(DecodeConfig),
|
||||
}
|
||||
|
||||
impl DecodeMode {
|
||||
/// Returns whether this mode will decrypt and decode received packets.
|
||||
#[must_use]
|
||||
pub fn should_decode(self) -> bool {
|
||||
matches!(self, DecodeMode::Decode(..))
|
||||
}
|
||||
|
||||
/// Returns whether this mode will decrypt received packets.
|
||||
#[must_use]
|
||||
pub fn should_decrypt(self) -> bool {
|
||||
self != DecodeMode::Pass
|
||||
}
|
||||
}
|
||||
|
||||
/// Configuration for [`DecodeMode::Decode`]
|
||||
#[non_exhaustive]
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash)]
|
||||
pub struct DecodeConfig {
|
||||
/// Configures the channel layout for output audio.
|
||||
///
|
||||
/// Defaults to [`Channels::Stereo`].
|
||||
pub channels: Channels,
|
||||
|
||||
/// Configures the sample rate for output audio.
|
||||
///
|
||||
/// Defaults to [`SampleRate::Hz48000`].
|
||||
pub sample_rate: SampleRate,
|
||||
}
|
||||
|
||||
impl DecodeConfig {
|
||||
/// Creates a new [`DecodeConfig`] with the specified channels and sample rate.
|
||||
#[must_use]
|
||||
pub fn new(channels: Channels, sample_rate: SampleRate) -> Self {
|
||||
Self {
|
||||
channels,
|
||||
sample_rate,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The channel layout of output audio when using [`DecodeMode::Decode`].
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash)]
|
||||
#[non_exhaustive]
|
||||
pub enum Channels {
|
||||
/// Decode received audio packets into a single channel.
|
||||
Mono,
|
||||
/// Decode received audio packets into two interleaved channels.
|
||||
///
|
||||
/// Received mono packets' samples will automatically be duplicated across
|
||||
/// both channels.
|
||||
///
|
||||
/// The default choice.
|
||||
#[default]
|
||||
Stereo,
|
||||
}
|
||||
|
||||
impl Channels {
|
||||
pub(crate) fn channels(self) -> usize {
|
||||
match self {
|
||||
Channels::Mono => 1,
|
||||
Channels::Stereo => 2,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Channels> for OpusChannels {
|
||||
fn from(value: Channels) -> Self {
|
||||
match value {
|
||||
Channels::Mono => OpusChannels::Mono,
|
||||
Channels::Stereo => OpusChannels::Stereo,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The sample rate of output audio when using [`DecodeMode::Decode`].
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash)]
|
||||
#[non_exhaustive]
|
||||
pub enum SampleRate {
|
||||
/// Decode to a sample rate of 8kHz.
|
||||
Hz8000,
|
||||
/// Decode to a sample rate of 12kHz.
|
||||
Hz12000,
|
||||
/// Decode to a sample rate of 16kHz.
|
||||
Hz16000,
|
||||
/// Decode to a sample rate of 24kHz.
|
||||
Hz24000,
|
||||
/// Decode to a sample rate of 48kHz.
|
||||
///
|
||||
/// The preferred option for encoding/decoding at or above CD quality.
|
||||
#[default]
|
||||
Hz48000,
|
||||
}
|
||||
|
||||
impl From<SampleRate> for u32 {
|
||||
fn from(value: SampleRate) -> Self {
|
||||
match value {
|
||||
SampleRate::Hz8000 => 8000,
|
||||
SampleRate::Hz12000 => 12000,
|
||||
SampleRate::Hz16000 => 16000,
|
||||
SampleRate::Hz24000 => 24000,
|
||||
SampleRate::Hz48000 => 48000,
|
||||
}
|
||||
}
|
||||
}
|
||||
59
app/vendor/songbird/src/driver/mix_mode.rs
vendored
59
app/vendor/songbird/src/driver/mix_mode.rs
vendored
@@ -1,59 +0,0 @@
|
||||
use opus2::Channels;
|
||||
use symphonia_core::audio::Layout;
|
||||
|
||||
use crate::constants::{MONO_FRAME_SIZE, STEREO_FRAME_SIZE};
|
||||
|
||||
/// Mixing behaviour for sent audio sources processed within the driver.
|
||||
///
|
||||
/// This has no impact on Opus packet passthrough, which will pass packets
|
||||
/// irrespective of their channel count.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum MixMode {
|
||||
/// Audio sources will be downmixed into a mono buffer.
|
||||
Mono,
|
||||
/// Audio sources will be mixed into into a stereo buffer, where mono sources
|
||||
/// will be duplicated into both channels.
|
||||
Stereo,
|
||||
}
|
||||
|
||||
impl MixMode {
|
||||
pub(crate) const fn to_opus(self) -> Channels {
|
||||
match self {
|
||||
Self::Mono => Channels::Mono,
|
||||
Self::Stereo => Channels::Stereo,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const fn sample_count_in_frame(self) -> usize {
|
||||
match self {
|
||||
Self::Mono => MONO_FRAME_SIZE,
|
||||
Self::Stereo => STEREO_FRAME_SIZE,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const fn channels(self) -> usize {
|
||||
match self {
|
||||
Self::Mono => 1,
|
||||
Self::Stereo => 2,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const fn symph_layout(self) -> Layout {
|
||||
match self {
|
||||
Self::Mono => Layout::Mono,
|
||||
Self::Stereo => Layout::Stereo,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<MixMode> for Layout {
|
||||
fn from(val: MixMode) -> Self {
|
||||
val.symph_layout()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<MixMode> for Channels {
|
||||
fn from(val: MixMode) -> Self {
|
||||
val.to_opus()
|
||||
}
|
||||
}
|
||||
357
app/vendor/songbird/src/driver/mod.rs
vendored
357
app/vendor/songbird/src/driver/mod.rs
vendored
@@ -1,357 +0,0 @@
|
||||
//! Runner for a voice connection.
|
||||
//!
|
||||
//! Songbird's driver is a mixed-sync system, using:
|
||||
//! * Asynchronous connection management, event-handling, and gateway integration.
|
||||
//! * Synchronous audio mixing, packet generation, and encoding.
|
||||
//!
|
||||
//! This splits up work according to its IO/compute bound nature, preventing packet
|
||||
//! generation from being slowed down past its deadline, or from affecting other
|
||||
//! asynchronous tasks your bot must handle.
|
||||
|
||||
#[cfg(feature = "internals")]
|
||||
pub mod bench_internals;
|
||||
|
||||
pub(crate) mod connection;
|
||||
mod crypto;
|
||||
#[cfg(feature = "receive")]
|
||||
mod decode_mode;
|
||||
mod mix_mode;
|
||||
pub mod retry;
|
||||
mod scheduler;
|
||||
pub(crate) mod tasks;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_config;
|
||||
#[cfg(any(test, feature = "internals"))]
|
||||
mod test_impls;
|
||||
|
||||
use connection::error::{Error, Result};
|
||||
pub use crypto::CryptoMode;
|
||||
pub(crate) use crypto::CryptoState;
|
||||
#[cfg(feature = "receive")]
|
||||
pub use decode_mode::*;
|
||||
pub use mix_mode::MixMode;
|
||||
pub use scheduler::{
|
||||
get_default_scheduler,
|
||||
Config as SchedulerConfig,
|
||||
Error as SchedulerError,
|
||||
LiveStatBlock,
|
||||
Mode as SchedulerMode,
|
||||
Scheduler,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub use test_config::*;
|
||||
#[cfg(any(test, feature = "internals"))]
|
||||
pub use test_impls::*;
|
||||
|
||||
#[cfg(feature = "builtin-queue")]
|
||||
use crate::tracks::TrackQueue;
|
||||
use crate::{
|
||||
events::EventData,
|
||||
input::Input,
|
||||
tracks::{Track, TrackHandle},
|
||||
Config,
|
||||
ConnectionInfo,
|
||||
Event,
|
||||
EventHandler,
|
||||
};
|
||||
use core::{
|
||||
future::Future,
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
use flume::{r#async::RecvFut, SendError, Sender};
|
||||
/// Opus encoder bitrate settings.
|
||||
pub use opus2::{self as opus, Bitrate};
|
||||
#[cfg(feature = "builtin-queue")]
|
||||
use std::time::Duration;
|
||||
#[allow(unused_imports)]
|
||||
pub use tasks::disposal::DisposalThread;
|
||||
use tasks::message::CoreMessage;
|
||||
use tracing::instrument;
|
||||
|
||||
/// The control object for a Discord voice connection, handling connection,
|
||||
/// mixing, encoding, en/decryption, and event generation.
|
||||
///
|
||||
/// When compiled with the `"builtin-queue"` feature, each driver includes a track queue
|
||||
/// as a convenience to prevent the additional overhead of per-guild state management.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Driver {
|
||||
config: Config,
|
||||
self_mute: bool,
|
||||
sender: Sender<CoreMessage>,
|
||||
// Making this an Option is an abhorrent hack to coerce the borrow checker
|
||||
// into letting us have an &TrackQueue at the same time as an &mut Driver.
|
||||
// This is probably preferable to cloning the driver: Arc<...> should be nonzero
|
||||
// and if the compiler's smart we'll just codegen a pointer swap. It definitely makes
|
||||
// use of NonZero.
|
||||
#[cfg(feature = "builtin-queue")]
|
||||
queue: Option<TrackQueue>,
|
||||
}
|
||||
|
||||
impl Driver {
|
||||
/// Creates a new voice driver.
|
||||
///
|
||||
/// This will create the core voice tasks in the background.
|
||||
#[inline]
|
||||
#[must_use]
|
||||
pub fn new(config: Config) -> Self {
|
||||
let sender = Self::start_inner(config.clone());
|
||||
|
||||
Driver {
|
||||
config,
|
||||
self_mute: false,
|
||||
sender,
|
||||
#[cfg(feature = "builtin-queue")]
|
||||
queue: Some(TrackQueue::default()),
|
||||
}
|
||||
}
|
||||
|
||||
fn start_inner(config: Config) -> Sender<CoreMessage> {
|
||||
let (tx, rx) = flume::unbounded();
|
||||
|
||||
tasks::start(config, rx, tx.clone());
|
||||
|
||||
tx
|
||||
}
|
||||
|
||||
fn restart_inner(&mut self) {
|
||||
self.sender = Self::start_inner(self.config.clone());
|
||||
|
||||
self.mute(self.self_mute);
|
||||
}
|
||||
|
||||
/// Connects to a voice channel using the specified server.
|
||||
///
|
||||
/// This method instantly contacts the driver tasks, and its
|
||||
/// does not need to be `await`ed to start the actual connection.
|
||||
#[instrument(skip(self))]
|
||||
pub fn connect(&mut self, info: ConnectionInfo) -> Connect {
|
||||
let (tx, rx) = flume::bounded(1);
|
||||
|
||||
self.raw_connect(info, tx);
|
||||
|
||||
Connect {
|
||||
inner: rx.into_recv_async(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Connects to a voice channel using the specified server.
|
||||
#[instrument(skip(self))]
|
||||
pub(crate) fn raw_connect(&mut self, info: ConnectionInfo, tx: Sender<Result<()>>) {
|
||||
self.send(CoreMessage::ConnectWithResult(info, tx));
|
||||
}
|
||||
|
||||
/// Leaves the current voice channel, disconnecting from it.
|
||||
///
|
||||
/// This does *not* forget settings, like whether to be self-deafened or
|
||||
/// self-muted.
|
||||
#[instrument(skip(self))]
|
||||
pub fn leave(&mut self) {
|
||||
self.send(CoreMessage::Disconnect);
|
||||
}
|
||||
|
||||
/// Sets whether the current connection is to be muted.
|
||||
///
|
||||
/// If there is no live voice connection, then this only acts as a settings
|
||||
/// update for future connections.
|
||||
#[instrument(skip(self))]
|
||||
pub fn mute(&mut self, mute: bool) {
|
||||
self.self_mute = mute;
|
||||
self.send(CoreMessage::Mute(mute));
|
||||
}
|
||||
|
||||
/// Returns whether the driver is muted (i.e., processes audio internally
|
||||
/// but submits none).
|
||||
#[instrument(skip(self))]
|
||||
pub fn is_mute(&self) -> bool {
|
||||
self.self_mute
|
||||
}
|
||||
|
||||
/// Plays audio from an input, returning a handle for further control.
|
||||
#[instrument(skip(self, input))]
|
||||
pub fn play_input(&mut self, input: Input) -> TrackHandle {
|
||||
self.play(input.into())
|
||||
}
|
||||
|
||||
/// Plays audio from an input, returning a handle for further control.
|
||||
///
|
||||
/// Unlike [`Self::play_input`], this stops all other inputs attached
|
||||
/// to the channel.
|
||||
#[instrument(skip(self, input))]
|
||||
pub fn play_only_input(&mut self, input: Input) -> TrackHandle {
|
||||
self.play_only(input.into())
|
||||
}
|
||||
|
||||
/// Plays audio from a [`Track`] object.
|
||||
///
|
||||
/// The main difference between this function and [`Self::play_input`] is
|
||||
/// that this allows for direct manipulation of the [`Track`] object
|
||||
/// before it is passed over to the voice and mixing contexts.
|
||||
#[instrument(skip(self, track))]
|
||||
pub fn play(&mut self, track: Track) -> TrackHandle {
|
||||
let (handle, ctx) = track.into_context();
|
||||
self.send(CoreMessage::AddTrack(Box::new(ctx)));
|
||||
|
||||
handle
|
||||
}
|
||||
|
||||
/// Exclusively plays audio from a [`Track`] object.
|
||||
///
|
||||
/// As in [`Self::play_only_input`], this stops all other sources attached to the
|
||||
/// channel. Like [`Self::play`], however, this allows for direct manipulation of the
|
||||
/// [`Track`] object before it is passed over to the voice and mixing contexts.
|
||||
#[instrument(skip(self, track))]
|
||||
pub fn play_only(&mut self, track: Track) -> TrackHandle {
|
||||
let (handle, ctx) = track.into_context();
|
||||
self.send(CoreMessage::SetTrack(Some(Box::new(ctx))));
|
||||
|
||||
handle
|
||||
}
|
||||
|
||||
/// Sets the bitrate for encoding Opus packets sent along
|
||||
/// the channel being managed.
|
||||
///
|
||||
/// The default rate is 128 kbps.
|
||||
/// Sensible values range between `Bits(512)` and `Bits(512_000)`
|
||||
/// bits per second.
|
||||
/// Alternatively, `Auto` and `Max` remain available.
|
||||
#[instrument(skip(self))]
|
||||
pub fn set_bitrate(&mut self, bitrate: Bitrate) {
|
||||
self.send(CoreMessage::SetBitrate(bitrate));
|
||||
}
|
||||
|
||||
/// Stops playing audio from all sources, if any are set.
|
||||
#[instrument(skip(self))]
|
||||
pub fn stop(&mut self) {
|
||||
self.send(CoreMessage::SetTrack(None));
|
||||
}
|
||||
|
||||
/// Sets the configuration for this driver (and parent `Call`, if applicable).
|
||||
#[instrument(skip(self))]
|
||||
pub fn set_config(&mut self, config: Config) {
|
||||
self.config = config.clone();
|
||||
self.send(CoreMessage::SetConfig(config));
|
||||
}
|
||||
|
||||
/// Returns a view of this driver's configuration.
|
||||
#[instrument(skip(self))]
|
||||
pub fn config(&self) -> &Config {
|
||||
&self.config
|
||||
}
|
||||
|
||||
/// Attach a global event handler to an audio context. Global events may receive
|
||||
/// any [`EventContext`].
|
||||
///
|
||||
/// Global timing events will tick regardless of whether audio is playing,
|
||||
/// so long as the bot is connected to a voice channel, and have no tracks.
|
||||
/// [`TrackEvent`]s will respond to all relevant tracks, giving some audio elements.
|
||||
///
|
||||
/// Users **must** ensure that no costly work or blocking occurs
|
||||
/// within the supplied function or closure. *Taking excess time could prevent
|
||||
/// timely sending of packets, causing audio glitches and delays*.
|
||||
///
|
||||
/// [`TrackEvent`]: crate::events::TrackEvent
|
||||
/// [`EventContext`]: crate::events::EventContext
|
||||
#[instrument(skip(self, action))]
|
||||
pub fn add_global_event<F: EventHandler + 'static>(&mut self, event: Event, action: F) {
|
||||
self.send(CoreMessage::AddEvent(EventData::new(event, action)));
|
||||
}
|
||||
|
||||
/// Removes all global event handlers from an audio context.
|
||||
#[instrument(skip(self))]
|
||||
pub fn remove_all_global_events(&mut self) {
|
||||
self.send(CoreMessage::RemoveGlobalEvents);
|
||||
}
|
||||
|
||||
/// Sends a message to the inner tasks, restarting it if necessary.
|
||||
fn send(&mut self, status: CoreMessage) {
|
||||
// Restart thread if it errored.
|
||||
if let Err(SendError(status)) = self.sender.send(status) {
|
||||
self.restart_inner();
|
||||
|
||||
self.sender.send(status).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "builtin-queue")]
|
||||
impl Driver {
|
||||
/// Returns a reference to this driver's built-in queue.
|
||||
///
|
||||
/// Requires the `"builtin-queue"` feature.
|
||||
/// Queue additions should be made via [`Driver::enqueue`] and
|
||||
/// [`Driver::enqueue_input`].
|
||||
#[must_use]
|
||||
pub fn queue(&self) -> &TrackQueue {
|
||||
self.queue
|
||||
.as_ref()
|
||||
.expect("Queue: The only case this can fail is if a previous queue operation panicked.")
|
||||
}
|
||||
|
||||
/// Adds an audio [`Input`] to this driver's built-in queue.
|
||||
///
|
||||
/// Requires the `"builtin-queue"` feature.
|
||||
pub async fn enqueue_input(&mut self, input: Input) -> TrackHandle {
|
||||
self.enqueue(input.into()).await
|
||||
}
|
||||
|
||||
/// Adds an existing [`Track`] to this driver's built-in queue.
|
||||
///
|
||||
/// Requires the `"builtin-queue"` feature.
|
||||
pub async fn enqueue(&mut self, mut track: Track) -> TrackHandle {
|
||||
let preload_time = TrackQueue::get_preload_time(&mut track).await;
|
||||
self.enqueue_with_preload(track, preload_time)
|
||||
}
|
||||
|
||||
/// Add an existing [`Track`] to the queue, using a known time to preload the next track.
|
||||
///
|
||||
/// See [`TrackQueue::add_with_preload`] for how `preload_time` is used.
|
||||
///
|
||||
/// Requires the `"builtin-queue"` feature.
|
||||
pub fn enqueue_with_preload(
|
||||
&mut self,
|
||||
track: Track,
|
||||
preload_time: Option<Duration>,
|
||||
) -> TrackHandle {
|
||||
let queue = self.queue.take().expect(
|
||||
"Enqueue: The only case this can fail is if a previous queue operation panicked.",
|
||||
);
|
||||
let handle = queue.add_with_preload(track, self, preload_time);
|
||||
self.queue = Some(queue);
|
||||
|
||||
handle
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Driver {
|
||||
fn default() -> Self {
|
||||
Self::new(Config::default())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Driver {
|
||||
/// Leaves the current connected voice channel, if connected to one, and
|
||||
/// forgets all configurations relevant to this Handler.
|
||||
fn drop(&mut self) {
|
||||
drop(self.sender.send(CoreMessage::Poison));
|
||||
}
|
||||
}
|
||||
|
||||
/// Future for a call to [`Driver::connect`].
|
||||
///
|
||||
/// This future awaits the *result* of a connection; the driver
|
||||
/// is messaged at the time of the call.
|
||||
pub struct Connect {
|
||||
inner: RecvFut<'static, Result<()>>,
|
||||
}
|
||||
|
||||
impl Future for Connect {
|
||||
type Output = Result<()>;
|
||||
|
||||
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
|
||||
match Pin::new(&mut self.inner).poll(cx) {
|
||||
Poll::Ready(r) => Poll::Ready(r.map_err(|_| Error::AttemptDiscarded).and_then(|x| x)),
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
}
|
||||
}
|
||||
51
app/vendor/songbird/src/driver/retry/mod.rs
vendored
51
app/vendor/songbird/src/driver/retry/mod.rs
vendored
@@ -1,51 +0,0 @@
|
||||
//! Configuration for connection retries.
|
||||
|
||||
mod strategy;
|
||||
|
||||
pub use self::strategy::*;
|
||||
|
||||
use std::num::NonZeroU8;
|
||||
|
||||
use crate::FloatDuration;
|
||||
|
||||
/// Configuration to be used for retrying driver connection attempts.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Retry {
|
||||
/// Strategy used to determine how long to wait between retry attempts.
|
||||
///
|
||||
/// *Defaults to an [`ExponentialBackoff`] from 0.25s
|
||||
/// to 10s, with a jitter of `0.1`.*
|
||||
///
|
||||
/// [`ExponentialBackoff`]: Strategy::Backoff
|
||||
pub strategy: Strategy,
|
||||
/// The maximum number of retries to attempt.
|
||||
///
|
||||
/// `None` will attempt an infinite number of retries,
|
||||
/// while `Some(0)` will attempt to connect *once* (no retries).
|
||||
///
|
||||
/// *Defaults to `Some(5)`.*
|
||||
pub retry_limit: Option<NonZeroU8>,
|
||||
}
|
||||
|
||||
impl Default for Retry {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
strategy: Strategy::Backoff(ExponentialBackoff::default()),
|
||||
retry_limit: Some(const { NonZeroU8::new(5).unwrap() }),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Retry {
|
||||
pub(crate) fn retry_in(
|
||||
&self,
|
||||
last_wait: Option<FloatDuration>,
|
||||
attempts: u8,
|
||||
) -> Option<FloatDuration> {
|
||||
if self.retry_limit.is_none_or(|a| attempts < a.get()) {
|
||||
Some(self.strategy.retry_in(last_wait))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
77
app/vendor/songbird/src/driver/retry/strategy.rs
vendored
77
app/vendor/songbird/src/driver/retry/strategy.rs
vendored
@@ -1,77 +0,0 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use rand::random;
|
||||
|
||||
use crate::FloatDuration;
|
||||
|
||||
/// Logic used to determine how long to wait between retry attempts.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
#[non_exhaustive]
|
||||
pub enum Strategy {
|
||||
/// The driver will wait for the same amount of time between each retry.
|
||||
Every(FloatDuration),
|
||||
/// Exponential backoff waiting strategy, where the duration between
|
||||
/// attempts (approximately) doubles each time.
|
||||
Backoff(ExponentialBackoff),
|
||||
}
|
||||
|
||||
impl Strategy {
|
||||
pub(crate) fn retry_in(&self, last_wait: Option<FloatDuration>) -> FloatDuration {
|
||||
match self {
|
||||
Self::Every(t) => *t,
|
||||
Self::Backoff(exp) => exp.retry_in(last_wait),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Exponential backoff waiting strategy.
|
||||
///
|
||||
/// Each attempt waits for twice the last delay plus/minus a
|
||||
/// random jitter, clamped to a min and max value.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct ExponentialBackoff {
|
||||
/// Minimum amount of time to wait between retries.
|
||||
///
|
||||
/// *Defaults to 0.25s.*
|
||||
pub min: FloatDuration,
|
||||
/// Maximum amount of time to wait between retries.
|
||||
///
|
||||
/// This will be clamped to `>=` min.
|
||||
///
|
||||
/// *Defaults to 10s.*
|
||||
pub max: FloatDuration,
|
||||
/// Amount of uniform random jitter to apply to generated wait times.
|
||||
/// I.e., 0.1 will add +/-10% to generated intervals.
|
||||
///
|
||||
/// This is restricted to within +/-100%.
|
||||
///
|
||||
/// *Defaults to `0.1`.*
|
||||
pub jitter: f32,
|
||||
}
|
||||
|
||||
impl Default for ExponentialBackoff {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
min: Duration::from_millis(250).into(),
|
||||
max: Duration::from_secs(10).into(),
|
||||
jitter: 0.1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ExponentialBackoff {
|
||||
pub(crate) fn retry_in(&self, last_wait: Option<FloatDuration>) -> FloatDuration {
|
||||
let min = self.min.as_secs_f32();
|
||||
let max = self.max.as_secs_f32();
|
||||
|
||||
let attempt = last_wait.map_or(min, |t| 2.0 * t.as_secs_f32());
|
||||
let perturb = (1.0 - (self.jitter * 2.0 * (random::<f32>() - 1.0))).clamp(0.0, 2.0);
|
||||
let mut target_time = attempt * perturb;
|
||||
|
||||
// Now clamp target time into given range.
|
||||
let safe_max = if max < min { min } else { max };
|
||||
target_time = target_time.clamp(min, safe_max);
|
||||
|
||||
FloatDuration::from_secs_f32(target_time)
|
||||
}
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
use super::*;
|
||||
|
||||
/// Configuration for how a [`Scheduler`] handles tasks.
|
||||
///
|
||||
/// [`Scheduler`]: super::Scheduler
|
||||
#[derive(Clone, Debug)]
|
||||
#[non_exhaustive]
|
||||
pub struct Config {
|
||||
/// How Live mixer tasks will be mapped to individual threads.
|
||||
///
|
||||
/// Defaults to `Mode::MaxPerThread(16)`.
|
||||
pub strategy: Mode,
|
||||
/// Move costly mixers to another thread if their parent worker is at
|
||||
/// risk of missing its deadlines.
|
||||
///
|
||||
/// Defaults to `true`.
|
||||
pub move_expensive_tasks: bool,
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
strategy: Mode::default(),
|
||||
move_expensive_tasks: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Strategies for mapping live mixer tasks to individual threads.
|
||||
///
|
||||
/// Defaults to `MaxPerThread(16)`.
|
||||
#[derive(Clone, Debug)]
|
||||
#[non_exhaustive]
|
||||
pub enum Mode {
|
||||
/// Allows at most `n` tasks to run per thread.
|
||||
MaxPerThread(NonZeroUsize),
|
||||
}
|
||||
|
||||
impl Mode {
|
||||
/// Returns the number of `Mixer`s that a scheduler should preallocate
|
||||
/// resources for.
|
||||
pub(crate) fn prealloc_size(&self) -> usize {
|
||||
match self {
|
||||
Self::MaxPerThread(n) => n.get(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the maximum number of concurrent mixers that a scheduler is
|
||||
/// allowed to place on a single thread.
|
||||
///
|
||||
/// Future scheduling modes may choose to limit *only* on execution cost.
|
||||
#[allow(clippy::unnecessary_wraps)]
|
||||
pub(crate) fn task_limit(&self) -> Option<usize> {
|
||||
match self {
|
||||
Self::MaxPerThread(n) => Some(n.get()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Mode {
|
||||
fn default() -> Self {
|
||||
Self::MaxPerThread(DEFAULT_MIXERS_PER_THREAD)
|
||||
}
|
||||
}
|
||||
335
app/vendor/songbird/src/driver/scheduler/idle.rs
vendored
335
app/vendor/songbird/src/driver/scheduler/idle.rs
vendored
@@ -1,335 +0,0 @@
|
||||
use std::{collections::HashMap, time::Duration};
|
||||
|
||||
use nohash_hasher::{BuildNoHashHasher, IntMap};
|
||||
use tokio::time::{Instant as TokInstant, Interval};
|
||||
use tracing::info;
|
||||
|
||||
use crate::constants::*;
|
||||
|
||||
use super::*;
|
||||
|
||||
const THREAD_CULL_TIMER: Duration = Duration::from_secs(60);
|
||||
|
||||
/// An async task responsible for maintaining UDP keepalives and event state for inactive
|
||||
/// `Mixer` tasks.
|
||||
pub(crate) struct Idle {
|
||||
config: Config,
|
||||
cull_timer: Duration,
|
||||
tasks: IntMap<TaskId, ParkedMixer>,
|
||||
// track taskids which are live to prevent their realloc? unlikely w u64 but still
|
||||
pub(crate) stats: Arc<StatBlock>,
|
||||
rx: Receiver<SchedulerMessage>,
|
||||
tx: Sender<SchedulerMessage>,
|
||||
next_id: TaskId,
|
||||
next_worker_id: WorkerId,
|
||||
workers: Vec<Worker>,
|
||||
to_cull: Vec<TaskId>,
|
||||
}
|
||||
|
||||
impl Idle {
|
||||
pub fn new(config: Config) -> (Self, Sender<SchedulerMessage>) {
|
||||
let (tx, rx) = flume::unbounded();
|
||||
|
||||
let stats = Arc::default();
|
||||
let tasks = HashMap::with_capacity_and_hasher(128, BuildNoHashHasher::default());
|
||||
|
||||
// TODO: include heap of keepalive sending times?
|
||||
let out = Self {
|
||||
config,
|
||||
cull_timer: THREAD_CULL_TIMER,
|
||||
tasks,
|
||||
stats,
|
||||
rx,
|
||||
tx: tx.clone(),
|
||||
next_id: TaskId::new(),
|
||||
next_worker_id: WorkerId::new(),
|
||||
workers: Vec::with_capacity(16),
|
||||
to_cull: vec![],
|
||||
};
|
||||
|
||||
(out, tx)
|
||||
}
|
||||
|
||||
/// Run the inner task until all external `Scheduler` handles are dropped.
|
||||
async fn run(&mut self) {
|
||||
let mut interval = tokio::time::interval(TIMESTEP_LENGTH);
|
||||
while self.run_once(&mut interval).await {}
|
||||
}
|
||||
|
||||
/// Run one 'tick' of idle thread maintenance.
|
||||
///
|
||||
/// This is a priority system over 2 main tasks:
|
||||
/// 1) handle scheduling/upgrade/action requests for mixers
|
||||
/// 2) [every 20ms]tick the main timer for each task, send keepalive if
|
||||
/// needed, reclaim & cull workers.
|
||||
///
|
||||
/// Idle mixers spawn an async task each to forward their `MixerMessage`s
|
||||
/// on to this task to be handled by 1). These tasks self-terminate if a
|
||||
/// message would make a mixer `now_live`.
|
||||
async fn run_once(&mut self, interval: &mut Interval) -> bool {
|
||||
tokio::select! {
|
||||
biased;
|
||||
msg = self.rx.recv_async() => match msg {
|
||||
Ok(SchedulerMessage::NewMixer(rx, ic, cfg)) => {
|
||||
let mut mixer = ParkedMixer::new(rx, ic, cfg);
|
||||
let id = self.next_id.incr();
|
||||
|
||||
mixer.spawn_forwarder(self.tx.clone(), id);
|
||||
self.tasks.insert(id, mixer);
|
||||
self.stats.add_idle_mixer();
|
||||
},
|
||||
Ok(SchedulerMessage::Demote(id, mut task)) => {
|
||||
task.send_gateway_not_speaking();
|
||||
|
||||
task.spawn_forwarder(self.tx.clone(), id);
|
||||
self.tasks.insert(id, task);
|
||||
},
|
||||
Ok(SchedulerMessage::Do(id, mix_msg)) => {
|
||||
let maybe_live = mix_msg.is_mixer_maybe_live();
|
||||
if let Some(task) = self.tasks.get_mut(&id) {
|
||||
match task.handle_message(mix_msg) {
|
||||
Ok(false) if maybe_live => {
|
||||
if task.mixer.tracks.is_empty() {
|
||||
// No tracks, likely due to SetConn.
|
||||
// Recreate message forwarding task.
|
||||
task.spawn_forwarder(self.tx.clone(), id);
|
||||
} else {
|
||||
let task = self.tasks.remove(&id).unwrap();
|
||||
self.schedule_mixer(task, id, None);
|
||||
}
|
||||
},
|
||||
Ok(false) => {},
|
||||
Ok(true) | Err(()) => self.to_cull.push(id),
|
||||
}
|
||||
} else {
|
||||
info!("Received post-cull message for {id:?}, discarding.");
|
||||
}
|
||||
},
|
||||
Ok(SchedulerMessage::Overspill(worker_id, id, task)) => {
|
||||
self.schedule_mixer(task, id, Some(worker_id));
|
||||
},
|
||||
Ok(SchedulerMessage::GetStats(tx)) => {
|
||||
_ = tx.send(self.workers.iter().map(Worker::stats).collect());
|
||||
},
|
||||
Ok(SchedulerMessage::Kill) | Err(_) => {
|
||||
return false;
|
||||
},
|
||||
},
|
||||
_ = interval.tick() => {
|
||||
// TODO: store keepalive sends in another data structure so
|
||||
// we don't check every task every 20ms.
|
||||
//
|
||||
// if we can also make tick handling lazy(er), we can also optimise for that.
|
||||
let now = TokInstant::now();
|
||||
|
||||
for (id, task) in &mut self.tasks {
|
||||
// NOTE: this is a non-blocking send so safe from async context.
|
||||
if task.tick_and_keepalive(now.into()).is_err() {
|
||||
self.to_cull.push(*id);
|
||||
}
|
||||
}
|
||||
|
||||
let mut i = 0;
|
||||
while i < self.workers.len() {
|
||||
if let Some(then) = self.workers[i].try_mark_empty(now) {
|
||||
if now.duration_since(then) >= self.cull_timer {
|
||||
self.workers.swap_remove(i);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
i += 1;
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
for id in self.to_cull.drain(..) {
|
||||
if let Some(tx) = self.tasks.remove(&id).and_then(|t| t.cull_handle) {
|
||||
_ = tx.send_async(()).await;
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
/// Promote a task to a live mixer thread.
|
||||
fn schedule_mixer(&mut self, mut task: ParkedMixer, id: TaskId, avoid: Option<WorkerId>) {
|
||||
if task.send_gateway_speaking().is_ok() {
|
||||
// If a worker ever completely fails, then we need to remove it here
|
||||
// `fetch_worker` will either find another, or generate us a new one if
|
||||
// none exist.
|
||||
|
||||
// We need to track ownership of the task coming back via SendError using this
|
||||
// Option.
|
||||
let mut loop_task = Some(task);
|
||||
loop {
|
||||
let task = loop_task.take().unwrap();
|
||||
let (worker, idx) = self.fetch_worker(&task, avoid);
|
||||
match worker.schedule_mixer(id, task) {
|
||||
Ok(()) => {
|
||||
self.stats.move_mixer_to_live();
|
||||
break;
|
||||
},
|
||||
Err(e) => {
|
||||
loop_task = Some(e.0 .1);
|
||||
let worker = self.workers.swap_remove(idx);
|
||||
|
||||
// NOTE: we have incremented worker's live counter for this mixer in
|
||||
// `schedule_mixer`.
|
||||
// The only time this branch is ever hit is if a worker crashed, so we
|
||||
// need to replicate some of their cleanup.
|
||||
self.stats
|
||||
.remove_live_mixers(worker.stats().live_mixers().saturating_sub(1));
|
||||
self.stats.remove_worker();
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch the first `Worker` that has room for a new task, creating one if needed.
|
||||
///
|
||||
/// If an inbound task has spilled from another thread, then do not reschedule it there.
|
||||
fn fetch_worker(
|
||||
&mut self,
|
||||
task: &ParkedMixer,
|
||||
avoid: Option<WorkerId>,
|
||||
) -> (&mut Worker, usize) {
|
||||
let idx = self
|
||||
.workers
|
||||
.iter()
|
||||
.position(|w| w.can_schedule(task, avoid))
|
||||
.unwrap_or_else(|| {
|
||||
self.workers.push(Worker::new(
|
||||
self.next_worker_id.incr(),
|
||||
self.config.clone(),
|
||||
self.tx.clone(),
|
||||
self.stats.clone(),
|
||||
));
|
||||
self.stats.add_worker();
|
||||
self.workers.len() - 1
|
||||
});
|
||||
|
||||
(&mut self.workers[idx], idx)
|
||||
}
|
||||
|
||||
pub fn spawn(mut self) {
|
||||
tokio::spawn(async move { self.run().await });
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
use crate::{
|
||||
constants::test_data::FILE_WEBM_TARGET,
|
||||
driver::{tasks::mixer::Mixer, OutputMode},
|
||||
input::File,
|
||||
Driver,
|
||||
};
|
||||
use tokio::runtime::Handle;
|
||||
|
||||
#[tokio::test]
|
||||
async fn inactive_mixers_dont_need_threads() {
|
||||
let sched = Scheduler::new(Config::default());
|
||||
let cfg = DriverConfig::default().scheduler(sched.clone());
|
||||
|
||||
let _drivers: Vec<Driver> = (0..1024).map(|_| Driver::new(cfg.clone())).collect();
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
|
||||
assert_eq!(sched.total_tasks(), 1024);
|
||||
assert_eq!(sched.live_tasks(), 0);
|
||||
assert_eq!(sched.worker_threads(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn active_mixers_spawn_threads() {
|
||||
let config = Config {
|
||||
strategy: Mode::default(),
|
||||
move_expensive_tasks: false,
|
||||
};
|
||||
|
||||
let sched = Scheduler::new(config);
|
||||
let (pkt_tx, _pkt_rx) = flume::unbounded();
|
||||
let cfg = DriverConfig::default()
|
||||
.scheduler(sched.clone())
|
||||
.override_connection(Some(OutputMode::Rtp(pkt_tx)));
|
||||
|
||||
let n_tasks = 1024;
|
||||
|
||||
let _drivers: Vec<Driver> = (0..n_tasks)
|
||||
.map(|_| {
|
||||
let mut driver = Driver::new(cfg.clone());
|
||||
let file = File::new(FILE_WEBM_TARGET);
|
||||
driver.play_input(file.into());
|
||||
driver
|
||||
})
|
||||
.collect();
|
||||
tokio::time::sleep(Duration::from_secs(10)).await;
|
||||
|
||||
assert_eq!(sched.total_tasks(), n_tasks);
|
||||
assert_eq!(sched.live_tasks(), n_tasks);
|
||||
assert_eq!(
|
||||
sched.worker_threads(),
|
||||
n_tasks / (DEFAULT_MIXERS_PER_THREAD.get() as u64)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn excess_threads_are_cleaned_up() {
|
||||
const TEST_TIMER: Duration = Duration::from_millis(500);
|
||||
|
||||
let config = Config {
|
||||
strategy: Mode::MaxPerThread(1.try_into().unwrap()),
|
||||
move_expensive_tasks: true,
|
||||
};
|
||||
|
||||
let (mut core, tx) = Idle::new(config.clone());
|
||||
core.cull_timer = TEST_TIMER;
|
||||
|
||||
let mut next_id = TaskId::new();
|
||||
let mut thread_id = WorkerId::new();
|
||||
let mut handles = vec![];
|
||||
for i in 0..2 {
|
||||
let mut worker = Worker::new(
|
||||
thread_id.incr(),
|
||||
config.clone(),
|
||||
tx.clone(),
|
||||
core.stats.clone(),
|
||||
);
|
||||
let ((mixer, listeners), track_handle) =
|
||||
Mixer::test_with_float_unending(Handle::current(), false);
|
||||
|
||||
let send_mixer = ParkedMixer {
|
||||
mixer: Box::new(mixer),
|
||||
ssrc: i,
|
||||
rtp_sequence: i as u16,
|
||||
rtp_timestamp: i,
|
||||
park_time: TokInstant::now().into(),
|
||||
last_cost: None,
|
||||
cull_handle: None,
|
||||
};
|
||||
core.stats.add_idle_mixer();
|
||||
core.stats.move_mixer_to_live();
|
||||
worker.schedule_mixer(next_id.incr(), send_mixer).unwrap();
|
||||
handles.push((track_handle, listeners));
|
||||
core.workers.push(worker);
|
||||
}
|
||||
|
||||
let mut timer = tokio::time::interval(TIMESTEP_LENGTH);
|
||||
assert!(core.run_once(&mut timer).await);
|
||||
|
||||
// Stop one of the handles, allow it to exit, and then run core again.
|
||||
handles[1].0.stop().unwrap();
|
||||
while core.workers[1].is_busy() {
|
||||
assert!(core.run_once(&mut timer).await);
|
||||
}
|
||||
|
||||
tokio::time::sleep(TEST_TIMER + Duration::from_secs(1)).await;
|
||||
while core.workers.len() != 1 {
|
||||
assert!(core.run_once(&mut timer).await);
|
||||
}
|
||||
|
||||
assert_eq!(core.stats.worker_threads(), 0);
|
||||
}
|
||||
}
|
||||
823
app/vendor/songbird/src/driver/scheduler/live.rs
vendored
823
app/vendor/songbird/src/driver/scheduler/live.rs
vendored
@@ -1,823 +0,0 @@
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use discortp::rtp::{MutableRtpPacket, RtpPacket};
|
||||
use flume::{SendError, TryRecvError};
|
||||
use tokio::time::Instant as TokInstant;
|
||||
|
||||
use crate::{
|
||||
constants::*,
|
||||
driver::tasks::{error::Error as DriverError, mixer::Mixer},
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
use crate::driver::test_config::TickStyle;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// The send-half of a worker thread, with bookkeeping mechanisms to help
|
||||
/// the idle task schedule incoming tasks.
|
||||
pub struct Worker {
|
||||
id: WorkerId,
|
||||
stats: Arc<LiveStatBlock>,
|
||||
config: Config,
|
||||
tx: Sender<(TaskId, ParkedMixer)>,
|
||||
known_empty_since: Option<TokInstant>,
|
||||
}
|
||||
|
||||
#[allow(missing_docs)]
|
||||
impl Worker {
|
||||
pub fn new(
|
||||
id: WorkerId,
|
||||
config: Config,
|
||||
sched_tx: Sender<SchedulerMessage>,
|
||||
global_stats: Arc<StatBlock>,
|
||||
) -> Self {
|
||||
let stats = Arc::new(LiveStatBlock::default());
|
||||
let (live_tx, live_rx) = flume::unbounded();
|
||||
|
||||
let core = Live::new(
|
||||
id,
|
||||
config.clone(),
|
||||
global_stats,
|
||||
stats.clone(),
|
||||
live_rx,
|
||||
sched_tx,
|
||||
);
|
||||
core.spawn();
|
||||
|
||||
Self {
|
||||
id,
|
||||
stats,
|
||||
config,
|
||||
tx: live_tx,
|
||||
known_empty_since: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Mark the worker thread as idle from the present time if it reports no tasks.
|
||||
///
|
||||
/// This time information is used for thread culling.
|
||||
#[inline]
|
||||
pub fn try_mark_empty(&mut self, now: TokInstant) -> Option<TokInstant> {
|
||||
if self.stats.live_mixers() == 0 {
|
||||
self.known_empty_since.get_or_insert(now);
|
||||
} else {
|
||||
self.mark_busy();
|
||||
}
|
||||
|
||||
self.known_empty_since
|
||||
}
|
||||
|
||||
/// Unset the thread culling time on this worker.
|
||||
#[inline]
|
||||
pub fn mark_busy(&mut self) {
|
||||
self.known_empty_since = None;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[inline]
|
||||
pub fn is_busy(&mut self) -> bool {
|
||||
self.known_empty_since.is_none()
|
||||
}
|
||||
|
||||
/// Return whether this thread has enough room (task count, spare cycles)
|
||||
/// for the given task.
|
||||
#[inline]
|
||||
#[must_use]
|
||||
pub fn can_schedule(&self, task: &ParkedMixer, avoid: Option<WorkerId>) -> bool {
|
||||
avoid.is_none_or(|id| !self.has_id(id)) && self.stats.has_room(&self.config.strategy, task)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[must_use]
|
||||
pub fn stats(&self) -> Arc<LiveStatBlock> {
|
||||
self.stats.clone()
|
||||
}
|
||||
|
||||
/// Increment this worker's statistics and hand off a task for execution.
|
||||
#[inline]
|
||||
pub fn schedule_mixer(
|
||||
&mut self,
|
||||
id: TaskId,
|
||||
task: ParkedMixer,
|
||||
) -> Result<(), SendError<(TaskId, ParkedMixer)>> {
|
||||
self.mark_busy();
|
||||
self.stats.add_mixer();
|
||||
self.tx.send((id, task))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn has_id(&self, id: WorkerId) -> bool {
|
||||
self.id == id
|
||||
}
|
||||
}
|
||||
|
||||
const PACKETS_PER_BLOCK: usize = 16;
|
||||
const MEMORY_CULL_TIMER: Duration = Duration::from_secs(10);
|
||||
|
||||
/// A synchronous thread responsible for mixing, encoding, encrypting, and
|
||||
/// sending the audio output of many `Mixer`s.
|
||||
///
|
||||
/// `Mixer`s remain `Box`ed due to large move costs, and unboxing them appeared to have
|
||||
/// a 5--10% perf cost from benchmarks.
|
||||
pub struct Live {
|
||||
packets: Vec<Box<[u8]>>,
|
||||
packet_lens: Vec<usize>,
|
||||
#[allow(clippy::vec_box)]
|
||||
tasks: Vec<Box<Mixer>>,
|
||||
ids: Vec<TaskId>,
|
||||
to_cull: Vec<bool>,
|
||||
|
||||
deadline: Instant,
|
||||
start_of_work: Option<Instant>,
|
||||
|
||||
id: WorkerId,
|
||||
config: Config,
|
||||
stats: Arc<LiveStatBlock>,
|
||||
global_stats: Arc<StatBlock>,
|
||||
rx: Receiver<(TaskId, ParkedMixer)>,
|
||||
tx: Sender<SchedulerMessage>,
|
||||
|
||||
excess_buffer_cull_time: Option<Instant>,
|
||||
}
|
||||
|
||||
#[allow(missing_docs)]
|
||||
impl Live {
|
||||
pub fn new(
|
||||
id: WorkerId,
|
||||
config: Config,
|
||||
global_stats: Arc<StatBlock>,
|
||||
stats: Arc<LiveStatBlock>,
|
||||
rx: Receiver<(TaskId, ParkedMixer)>,
|
||||
tx: Sender<SchedulerMessage>,
|
||||
) -> Self {
|
||||
let to_prealloc = config.strategy.prealloc_size();
|
||||
|
||||
let block_size = config
|
||||
.strategy
|
||||
.task_limit()
|
||||
.unwrap_or(PACKETS_PER_BLOCK)
|
||||
.min(PACKETS_PER_BLOCK);
|
||||
|
||||
let packets = vec![packet_block(block_size)];
|
||||
|
||||
Self {
|
||||
packets,
|
||||
packet_lens: Vec::with_capacity(to_prealloc),
|
||||
tasks: Vec::with_capacity(to_prealloc),
|
||||
ids: Vec::with_capacity(to_prealloc),
|
||||
to_cull: Vec::with_capacity(to_prealloc),
|
||||
|
||||
deadline: Instant::now(),
|
||||
start_of_work: None,
|
||||
|
||||
id,
|
||||
config,
|
||||
stats,
|
||||
global_stats,
|
||||
rx,
|
||||
tx,
|
||||
|
||||
excess_buffer_cull_time: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn run(&mut self) {
|
||||
while self.run_once() {}
|
||||
self.global_stats.remove_worker();
|
||||
}
|
||||
|
||||
/// Returns whether the loop should exit (i.e., culled by main `Scheduler`).
|
||||
#[inline]
|
||||
pub fn run_once(&mut self) -> bool {
|
||||
// Check for new tasks.
|
||||
if self.handle_scheduler_msgs().is_err() {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Receive commands for each task.
|
||||
self.handle_task_msgs();
|
||||
|
||||
// Move any idle calls back to the global pool.
|
||||
self.demote_and_remove_mixers();
|
||||
|
||||
// Take a clock measure before and after each packet.
|
||||
let mut pre_pkt_time = Instant::now();
|
||||
let mut worst_task = (0, Duration::default());
|
||||
|
||||
for (i, (packet_len, mixer)) in self
|
||||
.packet_lens
|
||||
.iter_mut()
|
||||
.zip(self.tasks.iter_mut())
|
||||
.enumerate()
|
||||
{
|
||||
let (block, inner) = get_memory_indices(i);
|
||||
match mixer.mix_and_build_packet(&mut self.packets[block][inner..][..VOICE_PACKET_MAX])
|
||||
{
|
||||
Ok(written_sz) => *packet_len = written_sz,
|
||||
e => {
|
||||
*packet_len = 0;
|
||||
rebuild_if_err(mixer, e, &mut self.to_cull, i);
|
||||
},
|
||||
}
|
||||
let post_pkt_time = Instant::now();
|
||||
let cost = post_pkt_time.duration_since(pre_pkt_time);
|
||||
if cost > worst_task.1 {
|
||||
worst_task = (i, cost);
|
||||
}
|
||||
pre_pkt_time = post_pkt_time;
|
||||
}
|
||||
|
||||
let end_of_work = pre_pkt_time;
|
||||
|
||||
if let Some(start_of_work) = self.start_of_work {
|
||||
let ns_cost = self.stats.store_compute_cost(end_of_work - start_of_work);
|
||||
|
||||
if self.config.move_expensive_tasks
|
||||
&& ns_cost >= RESCHEDULE_THRESHOLD
|
||||
&& self.ids.len() > 1
|
||||
{
|
||||
self.offload_mixer(worst_task.0, worst_task.1);
|
||||
}
|
||||
}
|
||||
|
||||
self.timed_remove_excess_blocks(end_of_work);
|
||||
|
||||
// Wait till the right time to send this packet:
|
||||
// usually a 20ms tick, in test modes this is either a finite number of runs or user input.
|
||||
self.march_deadline();
|
||||
|
||||
// Send all.
|
||||
self.start_of_work = Some(Instant::now());
|
||||
for (i, (packet_len, mixer)) in self
|
||||
.packet_lens
|
||||
.iter_mut()
|
||||
.zip(self.tasks.iter_mut())
|
||||
.enumerate()
|
||||
{
|
||||
let (block, inner) = get_memory_indices(i);
|
||||
let packet = &mut self.packets[block][inner..];
|
||||
if *packet_len > 0 {
|
||||
let res = mixer.send_packet(&packet[..*packet_len]);
|
||||
rebuild_if_err(mixer, res, &mut self.to_cull, i);
|
||||
}
|
||||
#[cfg(test)]
|
||||
if *packet_len == 0 {
|
||||
mixer.test_signal_empty_tick();
|
||||
}
|
||||
advance_rtp_counters(packet);
|
||||
}
|
||||
|
||||
for (i, mixer) in self.tasks.iter_mut().enumerate() {
|
||||
let res = mixer
|
||||
.audio_commands_events()
|
||||
.and_then(|()| mixer.check_and_send_keepalive(self.start_of_work));
|
||||
rebuild_if_err(mixer, res, &mut self.to_cull, i);
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn _march_deadline(&mut self) {
|
||||
// For testing, assume all will have same tick style.
|
||||
// Only count 'remaining loops' on one of the nodes.
|
||||
let mixer = self.tasks.get_mut(0).map(|m| {
|
||||
let style = m.config.tick_style.clone();
|
||||
(m, style)
|
||||
});
|
||||
|
||||
match mixer {
|
||||
None | Some((_, TickStyle::Timed)) => {
|
||||
std::thread::sleep(self.deadline.saturating_duration_since(Instant::now()));
|
||||
self.deadline += TIMESTEP_LENGTH;
|
||||
},
|
||||
Some((m, TickStyle::UntimedWithExecLimit(rx))) => {
|
||||
if m.remaining_loops.is_none() {
|
||||
if let Ok(new_val) = rx.recv() {
|
||||
m.remaining_loops = Some(new_val.wrapping_sub(1));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(cnt) = m.remaining_loops.as_mut() {
|
||||
if *cnt == 0 {
|
||||
m.remaining_loops = None;
|
||||
} else {
|
||||
*cnt = cnt.wrapping_sub(1);
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
#[inline(always)]
|
||||
#[allow(clippy::inline_always)]
|
||||
fn _march_deadline(&mut self) {
|
||||
std::thread::sleep(self.deadline.saturating_duration_since(Instant::now()));
|
||||
self.deadline += TIMESTEP_LENGTH;
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn march_deadline(&mut self) {
|
||||
#[cfg(feature = "internals")]
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
#[allow(clippy::used_underscore_items)]
|
||||
self._march_deadline();
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn handle_scheduler_msgs(&mut self) -> Result<(), ()> {
|
||||
let mut activation_time = None;
|
||||
loop {
|
||||
match self.rx.try_recv() {
|
||||
Ok((id, task)) => {
|
||||
self.add_task(
|
||||
task,
|
||||
id,
|
||||
*activation_time.get_or_insert_with(|| {
|
||||
self.deadline
|
||||
.checked_sub(TIMESTEP_LENGTH)
|
||||
.unwrap_or(self.deadline)
|
||||
}),
|
||||
);
|
||||
},
|
||||
Err(TryRecvError::Empty) => break,
|
||||
Err(TryRecvError::Disconnected) => return Err(()),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Handle messages from each tasks's `Driver`, marking dead tasks for removal.
|
||||
#[inline]
|
||||
fn handle_task_msgs(&mut self) {
|
||||
for (i, (packet, mixer)) in self
|
||||
.packets
|
||||
.iter_mut()
|
||||
.flat_map(|v| v.chunks_exact_mut(VOICE_PACKET_MAX))
|
||||
.zip(self.tasks.iter_mut())
|
||||
.enumerate()
|
||||
{
|
||||
let mut events_failure = false;
|
||||
let mut conn_failure = false;
|
||||
|
||||
let fatal = loop {
|
||||
match mixer.mix_rx.try_recv() {
|
||||
Ok(m) => {
|
||||
let (events, conn, should_exit) = mixer.handle_message(m, packet);
|
||||
events_failure |= events;
|
||||
conn_failure |= conn;
|
||||
|
||||
if should_exit {
|
||||
break true;
|
||||
}
|
||||
},
|
||||
Err(TryRecvError::Disconnected) => {
|
||||
break true;
|
||||
},
|
||||
|
||||
Err(TryRecvError::Empty) => {
|
||||
break false;
|
||||
},
|
||||
}
|
||||
};
|
||||
|
||||
if fatal || mixer.do_rebuilds(events_failure, conn_failure).is_err() {
|
||||
// this is not zipped in because it is *not* needed most ticks.
|
||||
self.to_cull[i] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "internals")]
|
||||
#[inline]
|
||||
pub fn mark_for_cull(&mut self, idx: usize) {
|
||||
self.to_cull[idx] = true;
|
||||
}
|
||||
|
||||
/// Check and demote for any tasks without live audio sources who have sent all
|
||||
/// necessary silent frames (or remove dead tasks).
|
||||
///
|
||||
/// This must occur *after* handling per-track events to prevent erroneously
|
||||
/// descheduling tasks.
|
||||
#[inline]
|
||||
pub fn demote_and_remove_mixers(&mut self) {
|
||||
let mut i = 0;
|
||||
while i < self.tasks.len() {
|
||||
#[cfg(test)]
|
||||
let force_conn = self.tasks[i].config.override_connection.is_some();
|
||||
#[cfg(not(test))]
|
||||
let force_conn = false;
|
||||
|
||||
// Benchmarking suggests that these asserts remove some bounds checks for us.
|
||||
assert!(i < self.tasks.len());
|
||||
assert!(i < self.to_cull.len());
|
||||
|
||||
if self.to_cull[i]
|
||||
|| (self.tasks[i].tracks.is_empty() && self.tasks[i].silence_frames == 0)
|
||||
|| !(self.tasks[i].conn_active.is_some() || force_conn)
|
||||
{
|
||||
self.stats.remove_mixer();
|
||||
|
||||
if let Some((id, parked)) = self.remove_task(i) {
|
||||
self.global_stats.move_mixer_to_idle();
|
||||
_ = self.tx.send(SchedulerMessage::Demote(id, parked));
|
||||
} else {
|
||||
self.global_stats.remove_live_mixer();
|
||||
}
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Return a given mixer to the main scheduler if this worker is overloaded.
|
||||
#[inline]
|
||||
pub fn offload_mixer(&mut self, idx: usize, cost: Duration) {
|
||||
self.stats.remove_mixer();
|
||||
|
||||
if let Some((id, mut parked)) = self.remove_task(idx) {
|
||||
self.global_stats.move_mixer_to_idle();
|
||||
parked.last_cost = Some(cost);
|
||||
_ = self
|
||||
.tx
|
||||
.send(SchedulerMessage::Overspill(self.id, id, parked));
|
||||
} else {
|
||||
self.global_stats.remove_live_mixer();
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn needed_blocks(&self) -> usize {
|
||||
let div = self.ids.len() / PACKETS_PER_BLOCK;
|
||||
let rem = self.ids.len() % PACKETS_PER_BLOCK;
|
||||
(rem != 0) as usize + div
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn has_excess_blocks(&self) -> bool {
|
||||
self.packets.len() > self.needed_blocks()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn remove_excess_blocks(&mut self) {
|
||||
self.packets.truncate(self.needed_blocks());
|
||||
}
|
||||
|
||||
/// Try to offload excess packet buffers.
|
||||
///
|
||||
/// If there is currently overallocation, then store the first time at which
|
||||
/// this was seenb. If this condition persists past `MEMORY_CULL_TIMER`, remove
|
||||
/// unnecessary blocks.
|
||||
#[inline]
|
||||
fn timed_remove_excess_blocks(&mut self, now: Instant) {
|
||||
if self.has_excess_blocks() {
|
||||
if let Some(mark_time) = self.excess_buffer_cull_time {
|
||||
if now.duration_since(mark_time) >= MEMORY_CULL_TIMER {
|
||||
self.remove_excess_blocks();
|
||||
self.excess_buffer_cull_time = None;
|
||||
}
|
||||
} else {
|
||||
self.excess_buffer_cull_time = Some(now);
|
||||
}
|
||||
} else {
|
||||
self.excess_buffer_cull_time = None;
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn add_task(&mut self, task: ParkedMixer, id: TaskId, activation_time: Instant) {
|
||||
let idx = self.ids.len();
|
||||
|
||||
let elapsed = task.park_time - activation_time;
|
||||
|
||||
let samples_f64 = elapsed.as_secs_f64() * (SAMPLE_RATE_RAW as f64);
|
||||
let mod_samples = (samples_f64 as u64) as u32;
|
||||
let rtp_timestamp = task.rtp_timestamp.wrapping_add(mod_samples);
|
||||
|
||||
self.ids.push(id);
|
||||
self.tasks.push(task.mixer);
|
||||
self.packet_lens.push(0);
|
||||
self.to_cull.push(false);
|
||||
|
||||
let (block, inner_idx) = get_memory_indices(idx);
|
||||
|
||||
while self.packets.len() <= block {
|
||||
self.add_packet_block();
|
||||
}
|
||||
let packet = &mut self.packets[block][inner_idx..][..VOICE_PACKET_MAX];
|
||||
|
||||
let mut rtp = MutableRtpPacket::new(packet).expect(
|
||||
"FATAL: Too few bytes in self.packet for RTP header.\
|
||||
(Blame: VOICE_PACKET_MAX?)",
|
||||
);
|
||||
rtp.set_ssrc(task.ssrc);
|
||||
rtp.set_timestamp(rtp_timestamp.into());
|
||||
rtp.set_sequence(task.rtp_sequence.into());
|
||||
}
|
||||
|
||||
/// Allocate and store a new packet block.
|
||||
///
|
||||
/// This will be full-size (`PACKETS_PER_BLOCK`) unless this block
|
||||
/// is a) the last required for the task limit and b) this limit
|
||||
/// is not aligned to `PACKETS_PER_BLOCK`.
|
||||
#[inline]
|
||||
fn add_packet_block(&mut self) {
|
||||
let n_packets = if let Some(limit) = self.config.strategy.task_limit() {
|
||||
let (block, inner) = get_memory_indices_unscaled(limit);
|
||||
if self.packets.len() < block || inner == 0 {
|
||||
PACKETS_PER_BLOCK
|
||||
} else {
|
||||
inner
|
||||
}
|
||||
} else {
|
||||
PACKETS_PER_BLOCK
|
||||
};
|
||||
self.packets.push(packet_block(n_packets));
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "internals"))]
|
||||
#[inline]
|
||||
pub fn add_task_direct(&mut self, task: Mixer, id: TaskId) {
|
||||
let id_0 = id.get();
|
||||
self.add_task(
|
||||
ParkedMixer {
|
||||
mixer: Box::new(task),
|
||||
ssrc: id_0 as u32,
|
||||
rtp_sequence: id_0 as u16,
|
||||
rtp_timestamp: id_0 as u32,
|
||||
park_time: Instant::now(),
|
||||
last_cost: None,
|
||||
cull_handle: None,
|
||||
},
|
||||
id,
|
||||
Instant::now(),
|
||||
);
|
||||
}
|
||||
|
||||
/// Remove a `Mixer`, returning it to the idle scheduler.
|
||||
///
|
||||
/// This operates by `swap_remove`ing each element of a Mixer's state, including
|
||||
/// on RTP packet headers. This is achieved by setting up a memcpy between
|
||||
/// buffer segments.
|
||||
#[inline]
|
||||
pub fn remove_task(&mut self, idx: usize) -> Option<(TaskId, ParkedMixer)> {
|
||||
let end = self.tasks.len() - 1;
|
||||
|
||||
let id = self.ids.swap_remove(idx);
|
||||
let _len = self.packet_lens.swap_remove(idx);
|
||||
let mixer = self.tasks.swap_remove(idx);
|
||||
let alive = !self.to_cull.swap_remove(idx);
|
||||
|
||||
let (block, inner_idx) = get_memory_indices(idx);
|
||||
|
||||
let (removed, replacement) = if end > idx {
|
||||
let (end_block, end_inner) = get_memory_indices(end);
|
||||
let (rest, target_block) = self.packets.split_at_mut(end_block);
|
||||
let (last_block, end_pkt) = target_block[0].split_at_mut(end_inner);
|
||||
|
||||
if end_block == block {
|
||||
(&mut last_block[inner_idx..], Some(end_pkt))
|
||||
} else {
|
||||
(&mut rest[block][inner_idx..], Some(end_pkt))
|
||||
}
|
||||
} else {
|
||||
(&mut self.packets[block][inner_idx..], None)
|
||||
};
|
||||
|
||||
let rtp = RtpPacket::new(removed).expect(
|
||||
"FATAL: Too few bytes in self.packet for RTP header.\
|
||||
(Blame: VOICE_PACKET_MAX?)",
|
||||
);
|
||||
let ssrc = rtp.get_ssrc();
|
||||
let rtp_timestamp = rtp.get_timestamp().into();
|
||||
let rtp_sequence = rtp.get_sequence().into();
|
||||
|
||||
if let Some(replacement) = replacement {
|
||||
// Copy the whole packet header since we know it'll be 4B aligned.
|
||||
// 'Just necessary fields' is 2B aligned.
|
||||
const COPY_LEN: usize = RtpPacket::minimum_packet_size();
|
||||
removed[..COPY_LEN].copy_from_slice(&replacement[..COPY_LEN]);
|
||||
}
|
||||
|
||||
alive.then(move || {
|
||||
let park_time = Instant::now();
|
||||
|
||||
(
|
||||
id,
|
||||
ParkedMixer {
|
||||
mixer,
|
||||
ssrc,
|
||||
rtp_sequence,
|
||||
rtp_timestamp,
|
||||
park_time,
|
||||
last_cost: None,
|
||||
cull_handle: None,
|
||||
},
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Spawn a new sync thread to manage `Mixer`s.
|
||||
fn spawn(mut self) {
|
||||
std::thread::spawn(move || {
|
||||
self.run();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Initialises a packet block of the required size, prefilling any constant RTP data.
|
||||
#[inline]
|
||||
fn packet_block(n_packets: usize) -> Box<[u8]> {
|
||||
let mut packets = vec![0u8; VOICE_PACKET_MAX * n_packets].into_boxed_slice();
|
||||
|
||||
for packet in packets.chunks_exact_mut(VOICE_PACKET_MAX) {
|
||||
let mut rtp = MutableRtpPacket::new(packet).expect(
|
||||
"FATAL: Too few bytes in self.packet for RTP header.\
|
||||
(Blame: VOICE_PACKET_MAX?)",
|
||||
);
|
||||
rtp.set_version(RTP_VERSION);
|
||||
rtp.set_payload_type(RTP_PROFILE_TYPE);
|
||||
}
|
||||
|
||||
packets
|
||||
}
|
||||
|
||||
/// Returns the block index into `self.packets` and the packet number in
|
||||
/// the block for a given worker's index.
|
||||
#[inline]
|
||||
fn get_memory_indices_unscaled(idx: usize) -> (usize, usize) {
|
||||
let block_size = PACKETS_PER_BLOCK;
|
||||
(idx / block_size, idx % block_size)
|
||||
}
|
||||
|
||||
/// Returns the block index into `self.packets` and the byte offset into
|
||||
/// a packet block for a given worker's index.
|
||||
#[inline]
|
||||
fn get_memory_indices(idx: usize) -> (usize, usize) {
|
||||
let (block, inner_unscaled) = get_memory_indices_unscaled(idx);
|
||||
(block, inner_unscaled * VOICE_PACKET_MAX)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn advance_rtp_counters(packet: &mut [u8]) {
|
||||
let mut rtp = MutableRtpPacket::new(packet).expect(
|
||||
"FATAL: Too few bytes in self.packet for RTP header.\
|
||||
(Blame: VOICE_PACKET_MAX?)",
|
||||
);
|
||||
rtp.set_sequence(rtp.get_sequence() + 1);
|
||||
rtp.set_timestamp(rtp.get_timestamp() + MONO_FRAME_SIZE as u32);
|
||||
}
|
||||
|
||||
/// Structured slightly confusingly: we only want to even access `cull_markers`
|
||||
/// in the event of error.
|
||||
#[inline]
|
||||
fn rebuild_if_err<T>(
|
||||
mixer: &mut Box<Mixer>,
|
||||
res: Result<T, DriverError>,
|
||||
cull_markers: &mut [bool],
|
||||
idx: usize,
|
||||
) {
|
||||
if let Err(e) = res {
|
||||
cull_markers[idx] |= mixer
|
||||
.do_rebuilds(
|
||||
e.should_trigger_interconnect_rebuild(),
|
||||
e.should_trigger_connect(),
|
||||
)
|
||||
.is_err();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
use crate::driver::test_impls::*;
|
||||
use tokio::runtime::Handle;
|
||||
|
||||
fn rtp_has_index(pkt: &[u8], sentinel_val: u16) {
|
||||
let rtp = RtpPacket::new(pkt).unwrap();
|
||||
|
||||
assert_eq!(rtp.get_version(), RTP_VERSION);
|
||||
assert_eq!(rtp.get_padding(), 0);
|
||||
assert_eq!(rtp.get_extension(), 0);
|
||||
assert_eq!(rtp.get_csrc_count(), 0);
|
||||
assert_eq!(rtp.get_marker(), 0);
|
||||
assert_eq!(rtp.get_payload_type(), RTP_PROFILE_TYPE);
|
||||
assert_eq!(rtp.get_sequence(), sentinel_val.into());
|
||||
assert_eq!(rtp.get_timestamp(), (sentinel_val as u32).into());
|
||||
assert_eq!(rtp.get_ssrc(), sentinel_val as u32);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn block_alloc_is_partial_small() {
|
||||
let n_mixers = 1;
|
||||
let (sched, _listeners) = MockScheduler::from_mixers(
|
||||
Some(Mode::MaxPerThread(n_mixers.try_into().unwrap())),
|
||||
(0..n_mixers)
|
||||
.map(|_| Mixer::test_with_float(1, Handle::current(), false))
|
||||
.collect(),
|
||||
);
|
||||
|
||||
assert_eq!(sched.core.packets.len(), 1);
|
||||
assert_eq!(sched.core.packets[0].len(), VOICE_PACKET_MAX);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn block_alloc_is_partial_large() {
|
||||
let n_mixers = 33;
|
||||
let (sched, _listeners) = MockScheduler::from_mixers(
|
||||
Some(Mode::MaxPerThread(n_mixers.try_into().unwrap())),
|
||||
(0..n_mixers)
|
||||
.map(|_| Mixer::test_with_float(1, Handle::current(), false))
|
||||
.collect(),
|
||||
);
|
||||
|
||||
assert_eq!(sched.core.packets.len(), 3);
|
||||
assert_eq!(
|
||||
sched.core.packets[0].len(),
|
||||
PACKETS_PER_BLOCK * VOICE_PACKET_MAX
|
||||
);
|
||||
assert_eq!(
|
||||
sched.core.packets[1].len(),
|
||||
PACKETS_PER_BLOCK * VOICE_PACKET_MAX
|
||||
);
|
||||
assert_eq!(sched.core.packets[2].len(), VOICE_PACKET_MAX);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deletion_moves_pkt_header() {
|
||||
let (mut sched, _listeners) = MockScheduler::from_mixers(
|
||||
None,
|
||||
(0..PACKETS_PER_BLOCK)
|
||||
.map(|_| Mixer::test_with_float(1, Handle::current(), false))
|
||||
.collect(),
|
||||
);
|
||||
|
||||
let last_idx = (PACKETS_PER_BLOCK - 1) as u16;
|
||||
|
||||
// Remove head.
|
||||
sched.core.remove_task(0);
|
||||
rtp_has_index(&sched.core.packets[0], last_idx);
|
||||
|
||||
// Remove head.
|
||||
sched.core.remove_task(5);
|
||||
rtp_has_index(&sched.core.packets[0][5 * VOICE_PACKET_MAX..], last_idx - 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deletion_moves_pkt_header_multiblock() {
|
||||
let n_pkts = PACKETS_PER_BLOCK + 8;
|
||||
let (mut sched, _listeners) = MockScheduler::from_mixers(
|
||||
None,
|
||||
(0..n_pkts)
|
||||
.map(|_| Mixer::test_with_float(1, Handle::current(), false))
|
||||
.collect(),
|
||||
);
|
||||
|
||||
let last_idx = (n_pkts - 1) as u16;
|
||||
|
||||
// Remove head (read from block 1 into block 0).
|
||||
sched.core.remove_task(0);
|
||||
rtp_has_index(&sched.core.packets[0], last_idx);
|
||||
|
||||
// Remove later (read from block 1 into block 1).
|
||||
sched.core.remove_task(17);
|
||||
rtp_has_index(&sched.core.packets[1][VOICE_PACKET_MAX..], last_idx - 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn packet_blocks_are_cleaned_up() {
|
||||
// Allocate 2 blocks.
|
||||
let n_pkts = PACKETS_PER_BLOCK + 1;
|
||||
let (mut sched, _listeners) = MockScheduler::from_mixers(
|
||||
None,
|
||||
(0..n_pkts)
|
||||
.map(|_| Mixer::test_with_float(1, Handle::current(), false))
|
||||
.collect(),
|
||||
);
|
||||
|
||||
// Assert no cleanup at start.
|
||||
assert!(sched.core.run_once());
|
||||
assert_eq!(sched.core.needed_blocks(), 2);
|
||||
assert!(sched.core.excess_buffer_cull_time.is_none());
|
||||
|
||||
// Remove only entry in last block. Cleanup should be sched'd.
|
||||
sched.core.remove_task(n_pkts - 1);
|
||||
assert!(sched.core.run_once());
|
||||
assert!(sched.core.has_excess_blocks());
|
||||
assert!(sched.core.excess_buffer_cull_time.is_some());
|
||||
|
||||
tokio::time::sleep(Duration::from_secs(2) + MEMORY_CULL_TIMER).await;
|
||||
|
||||
// Cleanup should be unsched'd.
|
||||
assert!(sched.core.run_once());
|
||||
assert!(sched.core.excess_buffer_cull_time.is_none());
|
||||
assert!(!sched.core.has_excess_blocks());
|
||||
}
|
||||
}
|
||||
180
app/vendor/songbird/src/driver/scheduler/mod.rs
vendored
180
app/vendor/songbird/src/driver/scheduler/mod.rs
vendored
@@ -1,180 +0,0 @@
|
||||
use std::{
|
||||
error::Error as StdError,
|
||||
fmt::Display,
|
||||
num::NonZeroUsize,
|
||||
sync::{Arc, OnceLock},
|
||||
};
|
||||
|
||||
use flume::{Receiver, RecvError, Sender};
|
||||
|
||||
use super::tasks::message::{Interconnect, MixerMessage};
|
||||
use crate::{constants::TIMESTEP_LENGTH, Config as DriverConfig};
|
||||
|
||||
mod config;
|
||||
mod idle;
|
||||
mod live;
|
||||
mod stats;
|
||||
mod task;
|
||||
|
||||
pub use config::*;
|
||||
use idle::*;
|
||||
pub use live::*;
|
||||
pub use stats::*;
|
||||
pub use task::*;
|
||||
|
||||
/// A soft maximum of 90% of the 20ms budget to account for variance in execution time.
|
||||
const RESCHEDULE_THRESHOLD: u64 = ((TIMESTEP_LENGTH.subsec_nanos() as u64) * 9) / 10;
|
||||
|
||||
const DEFAULT_MIXERS_PER_THREAD: NonZeroUsize = match NonZeroUsize::new(16) {
|
||||
Some(v) => v,
|
||||
None => unreachable!(),
|
||||
};
|
||||
|
||||
/// The default shared scheduler instance.
|
||||
///
|
||||
/// This is built using the default value of [`ScheduleMode`]. Users desiring
|
||||
/// a custom strategy should avoid calling [`Config::default`].
|
||||
///
|
||||
/// [`Config::default`]: crate::Config::default
|
||||
/// [`ScheduleMode`]: Mode
|
||||
pub fn get_default_scheduler() -> &'static Scheduler {
|
||||
static DEFAULT_SCHEDULER: OnceLock<Scheduler> = OnceLock::new();
|
||||
DEFAULT_SCHEDULER.get_or_init(Scheduler::default)
|
||||
}
|
||||
|
||||
/// A reference to a shared group of threads used for running idle and active
|
||||
/// audio threads.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Scheduler {
|
||||
inner: Arc<InnerScheduler>,
|
||||
}
|
||||
|
||||
/// Inner contents of a [`Scheduler`] instance.
|
||||
///
|
||||
/// This is an `Arc` around `Arc`'d contents so that we can make use of the
|
||||
/// drop check on `Scheduler` to clean up resources.
|
||||
#[derive(Clone, Debug)]
|
||||
struct InnerScheduler {
|
||||
tx: Sender<SchedulerMessage>,
|
||||
stats: Arc<StatBlock>,
|
||||
}
|
||||
|
||||
impl Scheduler {
|
||||
/// Create a new mixer scheduler from the allocation strategy in `config`.
|
||||
#[must_use]
|
||||
pub fn new(config: Config) -> Self {
|
||||
let (core, tx) = Idle::new(config);
|
||||
|
||||
let stats = core.stats.clone();
|
||||
core.spawn();
|
||||
|
||||
let inner = Arc::new(InnerScheduler { tx, stats });
|
||||
|
||||
Self { inner }
|
||||
}
|
||||
|
||||
pub(crate) fn new_mixer(
|
||||
&self,
|
||||
config: &DriverConfig,
|
||||
ic: Interconnect,
|
||||
rx: Receiver<MixerMessage>,
|
||||
) {
|
||||
self.inner
|
||||
.tx
|
||||
.send(SchedulerMessage::NewMixer(rx, ic, config.clone()))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Returns the total number of calls (idle and active) scheduled.
|
||||
#[must_use]
|
||||
pub fn total_tasks(&self) -> u64 {
|
||||
self.inner.stats.total_mixers()
|
||||
}
|
||||
|
||||
/// Returns the total number of *active* calls scheduled and processing
|
||||
/// audio.
|
||||
#[must_use]
|
||||
pub fn live_tasks(&self) -> u64 {
|
||||
self.inner.stats.live_mixers()
|
||||
}
|
||||
|
||||
/// Returns the total number of threads spawned to process live audio sessions.
|
||||
#[must_use]
|
||||
pub fn worker_threads(&self) -> u64 {
|
||||
self.inner.stats.worker_threads()
|
||||
}
|
||||
|
||||
/// Request a list of handles to statistics for currently live workers.
|
||||
pub async fn worker_thread_stats(&self) -> Result<Vec<Arc<LiveStatBlock>>, Error> {
|
||||
let (tx, rx) = flume::bounded(1);
|
||||
_ = self.inner.tx.send(SchedulerMessage::GetStats(tx));
|
||||
|
||||
rx.recv_async().await.map_err(Error::from)
|
||||
}
|
||||
|
||||
/// Request a list of handles to statistics for currently live workers with a blocking call.
|
||||
pub fn worker_thread_stats_blocking(&self) -> Result<Vec<Arc<LiveStatBlock>>, Error> {
|
||||
let (tx, rx) = flume::bounded(1);
|
||||
_ = self.inner.tx.send(SchedulerMessage::GetStats(tx));
|
||||
|
||||
rx.recv().map_err(Error::from)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for InnerScheduler {
|
||||
fn drop(&mut self) {
|
||||
_ = self.tx.send(SchedulerMessage::Kill);
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Scheduler {
|
||||
fn default() -> Self {
|
||||
Scheduler::new(Config::default())
|
||||
}
|
||||
}
|
||||
|
||||
/// Control messages for a scheduler.
|
||||
pub enum SchedulerMessage {
|
||||
/// Build a new `Mixer` as part of the initialisation of a `Driver`.
|
||||
NewMixer(Receiver<MixerMessage>, Interconnect, DriverConfig),
|
||||
/// Forward a command for
|
||||
Do(TaskId, MixerMessage),
|
||||
/// Return a `Mixer` from a worker back to the idle pool.
|
||||
Demote(TaskId, ParkedMixer),
|
||||
/// Move an expensive `Mixer` to another thread in the worker pool.
|
||||
Overspill(WorkerId, TaskId, ParkedMixer),
|
||||
/// Request a copy of all per-worker statistics.
|
||||
GetStats(Sender<Vec<Arc<LiveStatBlock>>>),
|
||||
/// Cleanup once all `Scheduler` handles are dropped.
|
||||
Kill,
|
||||
}
|
||||
|
||||
/// Errors encountered when communicating with the internals of a [`Scheduler`].
|
||||
///
|
||||
/// [`Scheduler`]: crate::driver::Scheduler
|
||||
#[non_exhaustive]
|
||||
#[derive(Debug)]
|
||||
pub enum Error {
|
||||
/// The scheduler exited or crashed while awating the request.
|
||||
Disconnected,
|
||||
}
|
||||
|
||||
impl Display for Error {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Disconnected => f.write_str("the scheduler terminated mid-request"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl StdError for Error {
|
||||
fn source(&self) -> Option<&(dyn StdError + 'static)> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
impl From<RecvError> for Error {
|
||||
fn from(_: RecvError) -> Self {
|
||||
Self::Disconnected
|
||||
}
|
||||
}
|
||||
134
app/vendor/songbird/src/driver/scheduler/stats.rs
vendored
134
app/vendor/songbird/src/driver/scheduler/stats.rs
vendored
@@ -1,134 +0,0 @@
|
||||
use std::{
|
||||
sync::atomic::{AtomicU64, Ordering},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use super::{Mode, ParkedMixer, RESCHEDULE_THRESHOLD};
|
||||
|
||||
/// Statistics shared by an entire `Scheduler`.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct StatBlock {
|
||||
total: AtomicU64,
|
||||
live: AtomicU64,
|
||||
threads: AtomicU64,
|
||||
}
|
||||
|
||||
#[allow(missing_docs)]
|
||||
impl StatBlock {
|
||||
#[inline]
|
||||
pub fn total_mixers(&self) -> u64 {
|
||||
self.total.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn live_mixers(&self) -> u64 {
|
||||
self.live.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn worker_threads(&self) -> u64 {
|
||||
self.threads.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn add_idle_mixer(&self) {
|
||||
self.total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn remove_idle_mixers(&self, n: u64) {
|
||||
self.total.fetch_sub(n, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn move_mixer_to_live(&self) {
|
||||
self.live.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn move_mixer_to_idle(&self) {
|
||||
self.move_mixers_to_idle(1);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn move_mixers_to_idle(&self, n: u64) {
|
||||
self.live.fetch_sub(n, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn remove_live_mixer(&self) {
|
||||
self.remove_live_mixers(1);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn remove_live_mixers(&self, n: u64) {
|
||||
self.move_mixers_to_idle(n);
|
||||
self.remove_idle_mixers(n);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn add_worker(&self) {
|
||||
self.threads.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn remove_worker(&self) {
|
||||
self.threads.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// Runtime statistics for an individual worker.
|
||||
///
|
||||
/// Individual statistics are measured atomically -- the worker thread
|
||||
/// may have been cleaned up, or its mixer count may not match the
|
||||
/// count when [`Self::last_compute_cost_ns`] was set.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct LiveStatBlock {
|
||||
live: AtomicU64,
|
||||
last_ns: AtomicU64,
|
||||
}
|
||||
|
||||
impl LiveStatBlock {
|
||||
/// Returns the number of mixer tasks scheduled on this worker thread.
|
||||
#[inline]
|
||||
pub fn live_mixers(&self) -> u64 {
|
||||
self.live.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn add_mixer(&self) {
|
||||
self.live.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn remove_mixer(&self) {
|
||||
self.live.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn store_compute_cost(&self, work: Duration) -> u64 {
|
||||
let cost = work.as_nanos() as u64;
|
||||
self.last_ns.store(cost, Ordering::Relaxed);
|
||||
cost
|
||||
}
|
||||
|
||||
/// Returns the number of nanoseconds required to process all worker threads'
|
||||
/// packet transmission, mixing, encoding, and encryption in the last tick.
|
||||
#[inline]
|
||||
pub fn last_compute_cost_ns(&self) -> u64 {
|
||||
self.last_ns.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn has_room(&self, strategy: &Mode, task: &ParkedMixer) -> bool {
|
||||
let task_room = strategy
|
||||
.task_limit()
|
||||
.is_none_or(|limit| self.live_mixers() < limit as u64);
|
||||
|
||||
let exec_room = task.last_cost.is_none_or(|cost| {
|
||||
cost.as_nanos() as u64 + self.last_compute_cost_ns() < RESCHEDULE_THRESHOLD
|
||||
});
|
||||
|
||||
task_room && exec_room
|
||||
}
|
||||
}
|
||||
214
app/vendor/songbird/src/driver/scheduler/task.rs
vendored
214
app/vendor/songbird/src/driver/scheduler/task.rs
vendored
@@ -1,214 +0,0 @@
|
||||
use std::{
|
||||
marker::PhantomData,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use flume::{Receiver, Sender};
|
||||
use nohash_hasher::IsEnabled;
|
||||
use rand::random;
|
||||
use tokio::runtime::Handle;
|
||||
|
||||
use crate::{
|
||||
driver::tasks::{
|
||||
error::Error as DriverError,
|
||||
message::{EventMessage, Interconnect, MixerMessage},
|
||||
mixer::Mixer,
|
||||
},
|
||||
Config,
|
||||
};
|
||||
|
||||
use super::SchedulerMessage;
|
||||
|
||||
/// Typesafe counter used to identify individual mixer/worker instances.
|
||||
#[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
|
||||
pub struct ResId<T>(u64, PhantomData<T>);
|
||||
#[allow(missing_docs)]
|
||||
pub type TaskId = ResId<TaskMarker>;
|
||||
#[allow(missing_docs)]
|
||||
pub type WorkerId = ResId<WorkerMarker>;
|
||||
|
||||
#[allow(missing_docs)]
|
||||
#[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
|
||||
pub struct TaskMarker;
|
||||
#[allow(missing_docs)]
|
||||
#[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
|
||||
pub struct WorkerMarker;
|
||||
|
||||
impl<T> IsEnabled for ResId<T> {}
|
||||
|
||||
#[allow(missing_docs)]
|
||||
impl<T: Copy> ResId<T> {
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn incr(&mut self) -> Self {
|
||||
let out = *self;
|
||||
self.0 = self.0.wrapping_add(1);
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "internals"))]
|
||||
#[must_use]
|
||||
pub fn get(self) -> u64 {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Copy> Default for ResId<T> {
|
||||
fn default() -> Self {
|
||||
Self(0, PhantomData)
|
||||
}
|
||||
}
|
||||
|
||||
/// An idle mixer instance, externally controlled by a `Driver`.
|
||||
///
|
||||
/// Since we do not allocate packet buffers for idle threads, this
|
||||
/// struct includes various RTP fields.
|
||||
pub struct ParkedMixer {
|
||||
/// Mixer, track, etc. state as well as message receivers.
|
||||
pub mixer: Box<Mixer>,
|
||||
/// The SSRC assigned to this voice session.
|
||||
pub ssrc: u32,
|
||||
/// The last recorded/generated RTP sequence.
|
||||
pub rtp_sequence: u16,
|
||||
/// The last recorded/generated RTP timestamp.
|
||||
pub rtp_timestamp: u32,
|
||||
/// The time at which this `Mixer` was made idle.
|
||||
///
|
||||
/// This is used when transitioning to a live state to determine
|
||||
/// how far we should adjust the RTP timestamp by.
|
||||
pub park_time: Instant,
|
||||
/// The last known cost of executing this task, if it had to be moved
|
||||
/// due to a limit on thread resources.
|
||||
pub last_cost: Option<Duration>,
|
||||
/// Handle to any forwarder task, used if this mixer is culled while idle.
|
||||
pub cull_handle: Option<Sender<()>>,
|
||||
}
|
||||
|
||||
#[allow(missing_docs)]
|
||||
impl ParkedMixer {
|
||||
/// Create a new `Mixer` in a parked state.
|
||||
#[must_use]
|
||||
pub fn new(mix_rx: Receiver<MixerMessage>, interconnect: Interconnect, config: Config) -> Self {
|
||||
Self {
|
||||
mixer: Box::new(Mixer::new(mix_rx, Handle::current(), interconnect, config)),
|
||||
ssrc: 0,
|
||||
rtp_sequence: random::<u16>(),
|
||||
rtp_timestamp: random::<u32>(),
|
||||
park_time: Instant::now(),
|
||||
last_cost: None,
|
||||
cull_handle: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawn a tokio task which forwards any mixer messages to the central `Idle` task pool.
|
||||
///
|
||||
/// Any requests which would cause this mixer to become live will terminate
|
||||
/// this task.
|
||||
pub fn spawn_forwarder(&mut self, tx: Sender<SchedulerMessage>, id: TaskId) {
|
||||
let (kill_tx, kill_rx) = flume::bounded(1);
|
||||
self.cull_handle = Some(kill_tx);
|
||||
|
||||
let remote_rx = self.mixer.mix_rx.clone();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = kill_rx.recv_async() => break,
|
||||
msg = remote_rx.recv_async() => {
|
||||
let exit = if let Ok(msg) = msg {
|
||||
let remove_self = msg.is_mixer_maybe_live();
|
||||
tx.send_async(SchedulerMessage::Do(id, msg)).await.is_err() || remove_self
|
||||
} else {
|
||||
true
|
||||
};
|
||||
|
||||
if exit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Returns whether the mixer should exit and be cleaned up.
|
||||
pub fn handle_message(&mut self, msg: MixerMessage) -> Result<bool, ()> {
|
||||
match msg {
|
||||
MixerMessage::SetConn(conn, ssrc) => {
|
||||
// Overridden because payload-specific fields are carried
|
||||
// externally on `ParkedMixer`.
|
||||
self.ssrc = ssrc;
|
||||
self.rtp_sequence = random::<u16>();
|
||||
self.rtp_timestamp = random::<u32>();
|
||||
self.mixer.conn_active = Some(conn);
|
||||
self.mixer.update_keepalive(ssrc);
|
||||
|
||||
Ok(false)
|
||||
},
|
||||
MixerMessage::Ws(ws) => {
|
||||
// Overridden so that we don't mistakenly tell Discord we're speaking.
|
||||
self.mixer.ws = ws;
|
||||
self.send_gateway_not_speaking();
|
||||
|
||||
Ok(false)
|
||||
},
|
||||
msg => {
|
||||
let (events_failure, conn_failure, should_exit) =
|
||||
self.mixer.handle_message(msg, &mut []);
|
||||
|
||||
self.mixer
|
||||
.do_rebuilds(events_failure, conn_failure)
|
||||
.map_err(|_| ())
|
||||
.map(|()| should_exit)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle periodic events attached to this `Mixer`, including timer state
|
||||
/// on the event thread and UDP keepalives needed to prevent session termination.
|
||||
///
|
||||
/// As we init our UDP sockets as non-blocking via Tokio -> `into_std`, it is
|
||||
/// safe to call UDP packet sends like this.
|
||||
pub fn tick_and_keepalive(&mut self, now: Instant) -> Result<(), ()> {
|
||||
// TODO: should we include an atomic which signals whether the event
|
||||
// thread *cares*, so we can prevent wakeups?
|
||||
// Can we do the same for live tracks?
|
||||
let mut events_failure = self.mixer.fire_event(EventMessage::Tick).is_err();
|
||||
|
||||
let ka_err = self
|
||||
.mixer
|
||||
.check_and_send_keepalive(Some(now))
|
||||
.or_else(DriverError::disarm_would_block);
|
||||
|
||||
let conn_failure = if let Err(e) = ka_err {
|
||||
events_failure |= e.should_trigger_interconnect_rebuild();
|
||||
e.should_trigger_connect()
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
self.mixer
|
||||
.do_rebuilds(events_failure, conn_failure)
|
||||
.map_err(|_| ())
|
||||
}
|
||||
|
||||
pub fn send_gateway_speaking(&mut self) -> Result<(), ()> {
|
||||
if let Err(e) = self.mixer.send_gateway_speaking() {
|
||||
self.mixer
|
||||
.do_rebuilds(
|
||||
e.should_trigger_interconnect_rebuild(),
|
||||
e.should_trigger_connect(),
|
||||
)
|
||||
.map_err(|_| ())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn send_gateway_not_speaking(&self) {
|
||||
self.mixer.send_gateway_not_speaking();
|
||||
}
|
||||
}
|
||||
44
app/vendor/songbird/src/driver/tasks/disposal.rs
vendored
44
app/vendor/songbird/src/driver/tasks/disposal.rs
vendored
@@ -1,44 +0,0 @@
|
||||
use super::message::*;
|
||||
use flume::{Receiver, Sender};
|
||||
use tracing::{instrument, trace};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DisposalThread(Sender<DisposalMessage>);
|
||||
|
||||
impl Default for DisposalThread {
|
||||
fn default() -> Self {
|
||||
Self::run()
|
||||
}
|
||||
}
|
||||
|
||||
impl DisposalThread {
|
||||
#[must_use]
|
||||
pub fn run() -> Self {
|
||||
let (mix_tx, mix_rx) = flume::unbounded();
|
||||
std::thread::spawn(move || {
|
||||
trace!("Disposal thread started.");
|
||||
runner(mix_rx);
|
||||
trace!("Disposal thread finished.");
|
||||
});
|
||||
|
||||
Self(mix_tx)
|
||||
}
|
||||
|
||||
pub(super) fn dispose(&self, message: DisposalMessage) {
|
||||
drop(self.0.send(message));
|
||||
}
|
||||
}
|
||||
|
||||
/// The mixer's disposal thread is also synchronous, due to tracks,
|
||||
/// inputs, etc. being based on synchronous I/O.
|
||||
///
|
||||
/// The mixer uses this to offload heavy and expensive drop operations
|
||||
/// to prevent deadline misses.
|
||||
#[instrument(skip(mix_rx))]
|
||||
#[expect(
|
||||
clippy::needless_pass_by_value,
|
||||
reason = "spawned on background thread, must take by value"
|
||||
)]
|
||||
fn runner(mix_rx: Receiver<DisposalMessage>) {
|
||||
while mix_rx.recv().is_ok() {}
|
||||
}
|
||||
150
app/vendor/songbird/src/driver/tasks/error.rs
vendored
150
app/vendor/songbird/src/driver/tasks/error.rs
vendored
@@ -1,150 +0,0 @@
|
||||
use super::message::*;
|
||||
use crate::ws::Error as WsError;
|
||||
use aes_gcm::Error as CryptoError;
|
||||
use davey::errors::EncryptError as DaveyEncryptError;
|
||||
use flume::SendError;
|
||||
use opus2::Error as OpusError;
|
||||
use std::io::{Error as IoError, ErrorKind as IoErrorKind};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Recipient {
|
||||
AuxNetwork,
|
||||
Event,
|
||||
Mixer,
|
||||
#[cfg(feature = "receive")]
|
||||
UdpRx,
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
#[derive(Debug)]
|
||||
#[non_exhaustive]
|
||||
pub enum Error {
|
||||
Crypto(CryptoError),
|
||||
DaveEncrypt,
|
||||
#[cfg(any(feature = "receive", test))]
|
||||
/// Received an illegal voice packet on the voice UDP socket.
|
||||
IllegalVoicePacket,
|
||||
InterconnectFailure(Recipient),
|
||||
Io(IoError),
|
||||
Other,
|
||||
}
|
||||
|
||||
impl Error {
|
||||
pub(crate) fn should_trigger_connect(&self) -> bool {
|
||||
match self {
|
||||
Error::InterconnectFailure(Recipient::AuxNetwork) => true,
|
||||
#[cfg(feature = "receive")]
|
||||
Error::InterconnectFailure(Recipient::UdpRx) => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn should_trigger_interconnect_rebuild(&self) -> bool {
|
||||
matches!(self, Error::InterconnectFailure(Recipient::Event))
|
||||
}
|
||||
|
||||
// This prevents a `WouldBlock` from triggering a full reconnect,
|
||||
// instead simply dropping the packet.
|
||||
pub(crate) fn disarm_would_block(self) -> Result<()> {
|
||||
match self {
|
||||
Self::Io(i) if i.kind() == IoErrorKind::WouldBlock => Ok(()),
|
||||
e => Err(e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<CryptoError> for Error {
|
||||
fn from(e: CryptoError) -> Self {
|
||||
Error::Crypto(e)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<DaveyEncryptError> for Error {
|
||||
fn from(_e: DaveyEncryptError) -> Self {
|
||||
Self::DaveEncrypt
|
||||
}
|
||||
}
|
||||
|
||||
impl From<IoError> for Error {
|
||||
fn from(e: IoError) -> Error {
|
||||
Error::Io(e)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<OpusError> for Error {
|
||||
fn from(_: OpusError) -> Error {
|
||||
Error::Other
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SendError<WsMessage>> for Error {
|
||||
fn from(_e: SendError<WsMessage>) -> Error {
|
||||
Error::InterconnectFailure(Recipient::AuxNetwork)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SendError<EventMessage>> for Error {
|
||||
fn from(_e: SendError<EventMessage>) -> Error {
|
||||
Error::InterconnectFailure(Recipient::Event)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SendError<MixerMessage>> for Error {
|
||||
fn from(_e: SendError<MixerMessage>) -> Error {
|
||||
Error::InterconnectFailure(Recipient::Mixer)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
impl From<SendError<UdpRxMessage>> for Error {
|
||||
fn from(_e: SendError<UdpRxMessage>) -> Error {
|
||||
Error::InterconnectFailure(Recipient::UdpRx)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<WsError> for Error {
|
||||
fn from(_: WsError) -> Error {
|
||||
Error::Other
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
#[non_exhaustive]
|
||||
pub enum DaveReinitError {
|
||||
Init,
|
||||
Reinit,
|
||||
Reset,
|
||||
CreateKeyPackage,
|
||||
Ws(WsError),
|
||||
}
|
||||
|
||||
impl From<davey::errors::InitError> for DaveReinitError {
|
||||
fn from(_e: davey::errors::InitError) -> Self {
|
||||
Self::Init
|
||||
}
|
||||
}
|
||||
|
||||
impl From<davey::errors::ReinitError> for DaveReinitError {
|
||||
fn from(_e: davey::errors::ReinitError) -> Self {
|
||||
Self::Reinit
|
||||
}
|
||||
}
|
||||
|
||||
impl From<davey::errors::ResetError> for DaveReinitError {
|
||||
fn from(_e: davey::errors::ResetError) -> Self {
|
||||
Self::Reset
|
||||
}
|
||||
}
|
||||
|
||||
impl From<davey::errors::CreateKeyPackageError> for DaveReinitError {
|
||||
fn from(_e: davey::errors::CreateKeyPackageError) -> Self {
|
||||
Self::CreateKeyPackage
|
||||
}
|
||||
}
|
||||
|
||||
impl From<WsError> for DaveReinitError {
|
||||
fn from(e: WsError) -> Self {
|
||||
Self::Ws(e)
|
||||
}
|
||||
}
|
||||
139
app/vendor/songbird/src/driver/tasks/events.rs
vendored
139
app/vendor/songbird/src/driver/tasks/events.rs
vendored
@@ -1,139 +0,0 @@
|
||||
use super::message::*;
|
||||
use crate::{
|
||||
events::{EventStore, GlobalEvents, TrackEvent},
|
||||
tracks::{ReadyState, TrackHandle, TrackState},
|
||||
};
|
||||
use flume::Receiver;
|
||||
use tracing::{debug, info, instrument, trace};
|
||||
|
||||
#[instrument(skip(evt_rx))]
|
||||
pub(crate) async fn runner(evt_rx: Receiver<EventMessage>) {
|
||||
let mut global = GlobalEvents::default();
|
||||
|
||||
let mut events: Vec<EventStore> = vec![];
|
||||
let mut states: Vec<TrackState> = vec![];
|
||||
let mut handles: Vec<TrackHandle> = vec![];
|
||||
|
||||
while let Ok(msg) = evt_rx.recv_async().await {
|
||||
match msg {
|
||||
EventMessage::AddGlobalEvent(data) => {
|
||||
info!("Global event added.");
|
||||
global.add_event(data);
|
||||
},
|
||||
EventMessage::AddTrackEvent(i, data) => {
|
||||
info!("Adding event to track {}.", i);
|
||||
|
||||
let event_store = events
|
||||
.get_mut(i)
|
||||
.expect("Event thread was given an illegal store index for AddTrackEvent.");
|
||||
let state = states
|
||||
.get_mut(i)
|
||||
.expect("Event thread was given an illegal state index for AddTrackEvent.");
|
||||
|
||||
event_store.add_event(data, state.position);
|
||||
},
|
||||
EventMessage::FireCoreEvent(ctx) => {
|
||||
let ctx = ctx.to_user_context();
|
||||
let evt = ctx
|
||||
.to_core_event()
|
||||
.expect("Event thread was passed a non-core event in FireCoreEvent.");
|
||||
|
||||
trace!("Firing core event {:?}.", evt);
|
||||
|
||||
global.fire_core_event(evt, ctx).await;
|
||||
},
|
||||
EventMessage::RemoveGlobalEvents => {
|
||||
global.remove_handlers();
|
||||
},
|
||||
EventMessage::AddTrack(store, state, handle) => {
|
||||
events.push(store);
|
||||
states.push(state);
|
||||
handles.push(handle);
|
||||
|
||||
info!("Event state for track {} added", events.len());
|
||||
},
|
||||
EventMessage::ChangeState(i, change) => {
|
||||
let max_states = states.len();
|
||||
debug!(
|
||||
"Changing state for track {} of {}: {:?}",
|
||||
i, max_states, change
|
||||
);
|
||||
|
||||
let state = states
|
||||
.get_mut(i)
|
||||
.expect("Event thread was given an illegal state index for ChangeState.");
|
||||
|
||||
match change {
|
||||
TrackStateChange::Mode(mut mode) => {
|
||||
std::mem::swap(&mut state.playing, &mut mode);
|
||||
if state.playing != mode {
|
||||
global.fire_track_event(state.playing.as_track_event(), i);
|
||||
if let Some(other_evts) = state.playing.also_fired_track_events() {
|
||||
for evt in other_evts {
|
||||
global.fire_track_event(evt, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
TrackStateChange::Volume(vol) => {
|
||||
state.volume = vol;
|
||||
},
|
||||
TrackStateChange::Position(pos) => {
|
||||
// Currently, only Tick should fire time events.
|
||||
state.position = pos;
|
||||
},
|
||||
TrackStateChange::Loops(loops, user_set) => {
|
||||
state.loops = loops;
|
||||
if !user_set {
|
||||
global.fire_track_event(TrackEvent::Loop, i);
|
||||
}
|
||||
},
|
||||
TrackStateChange::Total(new) => {
|
||||
// Massive, unprecedented state changes.
|
||||
*state = new;
|
||||
},
|
||||
TrackStateChange::Ready(ready_state) => {
|
||||
state.ready = ready_state;
|
||||
|
||||
match ready_state {
|
||||
ReadyState::Playable => {
|
||||
global.fire_track_event(TrackEvent::Playable, i);
|
||||
},
|
||||
ReadyState::Preparing => {
|
||||
global.fire_track_event(TrackEvent::Preparing, i);
|
||||
},
|
||||
ReadyState::Uninitialised => {},
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
EventMessage::RemoveAllTracks => {
|
||||
info!("Event state for all tracks removed.");
|
||||
|
||||
events.clear();
|
||||
states.clear();
|
||||
handles.clear();
|
||||
},
|
||||
EventMessage::Tick => {
|
||||
// NOTE: this should fire saved up blocks of state change evts.
|
||||
global.tick(&mut events, &mut states, &mut handles).await;
|
||||
|
||||
let mut i = 0;
|
||||
while i < states.len() {
|
||||
if states[i].playing.is_done() {
|
||||
info!("Event state for track {} of {} removed.", i, events.len());
|
||||
|
||||
events.swap_remove(i);
|
||||
states.swap_remove(i);
|
||||
handles.swap_remove(i);
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
},
|
||||
EventMessage::Poison => break,
|
||||
}
|
||||
}
|
||||
|
||||
trace!("Event thread exited.");
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
#![allow(missing_docs)]
|
||||
|
||||
use crate::{
|
||||
driver::{connection::error::Error, Bitrate, Config},
|
||||
events::{context_data::DisconnectReason, EventData},
|
||||
tracks::{Track, TrackCommand, TrackHandle},
|
||||
ConnectionInfo,
|
||||
};
|
||||
use flume::{Receiver, Sender};
|
||||
|
||||
pub enum CoreMessage {
|
||||
ConnectWithResult(ConnectionInfo, Sender<Result<(), Error>>),
|
||||
RetryConnect(usize),
|
||||
SignalWsClosure(usize, ConnectionInfo, Option<DisconnectReason>),
|
||||
Disconnect,
|
||||
SetTrack(Option<Box<TrackContext>>),
|
||||
AddTrack(Box<TrackContext>),
|
||||
SetBitrate(Bitrate),
|
||||
AddEvent(EventData),
|
||||
RemoveGlobalEvents,
|
||||
SetConfig(Config),
|
||||
Mute(bool),
|
||||
Reconnect,
|
||||
FullReconnect,
|
||||
RebuildInterconnect,
|
||||
Poison,
|
||||
}
|
||||
|
||||
pub struct TrackContext {
|
||||
pub track: Track,
|
||||
pub handle: TrackHandle,
|
||||
pub receiver: Receiver<TrackCommand>,
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
#![allow(missing_docs)]
|
||||
|
||||
use crate::{driver::tasks::mixer::InternalTrack, tracks::TrackHandle};
|
||||
|
||||
#[allow(dead_code)] // We don't read because all we are doing is dropping.
|
||||
pub enum DisposalMessage {
|
||||
Track(Box<InternalTrack>),
|
||||
Handle(TrackHandle),
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
#![allow(missing_docs)]
|
||||
|
||||
use crate::{
|
||||
events::{CoreContext, EventData, EventStore},
|
||||
tracks::{LoopState, PlayMode, ReadyState, TrackHandle, TrackState},
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
pub enum EventMessage {
|
||||
// Event related.
|
||||
// Track events should fire off the back of state changes.
|
||||
AddGlobalEvent(EventData),
|
||||
AddTrackEvent(usize, EventData),
|
||||
FireCoreEvent(CoreContext),
|
||||
RemoveGlobalEvents,
|
||||
|
||||
AddTrack(EventStore, TrackState, TrackHandle),
|
||||
ChangeState(usize, TrackStateChange),
|
||||
RemoveAllTracks,
|
||||
Tick,
|
||||
|
||||
Poison,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum TrackStateChange {
|
||||
Mode(PlayMode),
|
||||
Volume(f32),
|
||||
Position(Duration),
|
||||
// Bool indicates user-set.
|
||||
Loops(LoopState, bool),
|
||||
Total(TrackState),
|
||||
Ready(ReadyState),
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
#![allow(missing_docs)]
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
use super::UdpRxMessage;
|
||||
use super::{Interconnect, TrackContext, WsMessage};
|
||||
|
||||
use crate::{
|
||||
driver::{crypto::Cipher, Bitrate, Config, CryptoState},
|
||||
input::{AudioStreamError, Compose, Parsed},
|
||||
};
|
||||
use flume::Sender;
|
||||
use std::{
|
||||
net::UdpSocket,
|
||||
sync::{atomic::AtomicU16, Arc, RwLock},
|
||||
};
|
||||
use symphonia_core::{errors::Error as SymphoniaError, formats::SeekedTo};
|
||||
|
||||
pub struct MixerConnection {
|
||||
pub cipher: Cipher,
|
||||
pub crypto_state: CryptoState,
|
||||
pub dave_session: Arc<RwLock<Option<davey::DaveSession>>>,
|
||||
pub dave_protocol_version: Arc<AtomicU16>,
|
||||
#[cfg(feature = "receive")]
|
||||
pub udp_rx: Sender<UdpRxMessage>,
|
||||
pub udp_tx: UdpSocket,
|
||||
}
|
||||
|
||||
pub enum MixerMessage {
|
||||
AddTrack(Box<TrackContext>),
|
||||
SetTrack(Option<Box<TrackContext>>),
|
||||
|
||||
SetBitrate(Bitrate),
|
||||
SetConfig(Config),
|
||||
SetMute(bool),
|
||||
|
||||
SetConn(MixerConnection, u32),
|
||||
Ws(Option<Sender<WsMessage>>),
|
||||
DropConn,
|
||||
|
||||
ReplaceInterconnect(Interconnect),
|
||||
RebuildEncoder,
|
||||
|
||||
Poison,
|
||||
}
|
||||
|
||||
impl MixerMessage {
|
||||
#[must_use]
|
||||
pub fn is_mixer_maybe_live(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::AddTrack(_) | Self::SetTrack(Some(_)) | Self::SetConn(..)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub enum MixerInputResultMessage {
|
||||
CreateErr(Arc<AudioStreamError>),
|
||||
ParseErr(Arc<SymphoniaError>),
|
||||
Seek(
|
||||
Parsed,
|
||||
Option<Box<dyn Compose>>,
|
||||
Result<SeekedTo, Arc<SymphoniaError>>,
|
||||
),
|
||||
Built(Parsed, Option<Box<dyn Compose>>),
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
#![allow(missing_docs)]
|
||||
|
||||
mod core;
|
||||
mod disposal;
|
||||
mod events;
|
||||
mod mixer;
|
||||
#[cfg(feature = "receive")]
|
||||
mod udp_rx;
|
||||
mod ws;
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
pub use self::udp_rx::*;
|
||||
pub use self::{core::*, disposal::*, events::*, mixer::*, ws::*};
|
||||
|
||||
use flume::Sender;
|
||||
use tokio::spawn;
|
||||
use tracing::trace;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Interconnect {
|
||||
pub core: Sender<CoreMessage>,
|
||||
pub events: Sender<EventMessage>,
|
||||
pub mixer: Sender<MixerMessage>,
|
||||
}
|
||||
|
||||
impl Interconnect {
|
||||
pub fn poison(&self) {
|
||||
drop(self.events.send(EventMessage::Poison));
|
||||
}
|
||||
|
||||
pub fn poison_all(&self) {
|
||||
drop(self.mixer.send(MixerMessage::Poison));
|
||||
self.poison();
|
||||
}
|
||||
|
||||
pub fn restart_volatile_internals(&mut self) {
|
||||
self.poison();
|
||||
|
||||
let (evt_tx, evt_rx) = flume::unbounded();
|
||||
|
||||
self.events = evt_tx;
|
||||
|
||||
spawn(async move {
|
||||
trace!("Event processor restarted.");
|
||||
super::events::runner(evt_rx).await;
|
||||
trace!("Event processor finished.");
|
||||
});
|
||||
|
||||
// Make mixer aware of new targets...
|
||||
drop(
|
||||
self.mixer
|
||||
.send(MixerMessage::ReplaceInterconnect(self.clone())),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
#![allow(missing_docs)]
|
||||
|
||||
use super::Interconnect;
|
||||
use crate::driver::DecodeConfig;
|
||||
use dashmap::{DashMap, DashSet};
|
||||
use serenity_voice_model::id::UserId;
|
||||
|
||||
pub enum UdpRxMessage {
|
||||
SetConfig(DecodeConfig),
|
||||
ReplaceInterconnect(Interconnect),
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct SsrcTracker {
|
||||
pub disconnected_users: DashSet<UserId>,
|
||||
pub user_ssrc_map: DashMap<UserId, u32>,
|
||||
pub ssrc_user_map: DashMap<u32, UserId>,
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
#![allow(missing_docs)]
|
||||
|
||||
use super::Interconnect;
|
||||
use crate::{model::Event as GatewayEvent, ws::WsStream};
|
||||
|
||||
pub enum WsMessage {
|
||||
Ws(Box<WsStream>),
|
||||
ReplaceInterconnect(Interconnect),
|
||||
SetKeepalive(f64),
|
||||
Speaking(bool),
|
||||
Deliver(GatewayEvent),
|
||||
}
|
||||
@@ -1,446 +0,0 @@
|
||||
use super::*;
|
||||
|
||||
/// Mix a track's audio stream into either the shared mixing buffer, or directly into the output
|
||||
/// packet ("passthrough") when possible.
|
||||
///
|
||||
/// Passthrough is highest performance, but the source MUST be opus, have 20ms frames, and be the only
|
||||
/// live track. In this case we copy the opus-encoded data with no changes. Otherwise, we fall back to
|
||||
/// below.
|
||||
///
|
||||
/// There are a few functional requirements here for non-passthrough mixing that make it tricky:
|
||||
/// * Input frame lengths are not congruent with what we need to send (i.e., 26.12ms in MP3 vs
|
||||
/// needed 20ms).
|
||||
/// * Input audio arrives at a different sample rate from required (i.e., 44.1 vs needed 48 kHz).
|
||||
/// * Input data may not be `f32`s.
|
||||
/// * Input data may not match stereo/mono of desired output.
|
||||
///
|
||||
/// All of the above challenges often happen at once. The rough pipeline in processing is:
|
||||
///
|
||||
/// until source end or 20 ms taken:
|
||||
/// (use previous frame 'til empty / get new frame) -> [resample] -> [audio += vol * (sample as f32)]
|
||||
///
|
||||
/// Typically, we mix between a subset of the input packet and the output buf because the 20ms window
|
||||
/// straddles packet boundaries. If there's enough space AND 48kHz AND receive f32s, then we use a fast
|
||||
/// path.
|
||||
///
|
||||
/// In the mono -> stereo case, we duplicate across all target channels. In stereo -> mono, we average
|
||||
/// the samples from each channel.
|
||||
///
|
||||
/// To avoid needing to hold onto resampled data longer than one mix cycle, we take enough input samples
|
||||
/// to fill a chunk of the mixer (e.g., 10ms == 20ms / 2) so that they will all be used.
|
||||
///
|
||||
/// This is a fairly annoying piece of code to reason about, mainly because you need to hold so many
|
||||
/// internal positions into: the mix buffer, resample buffers, and previous/current packets
|
||||
/// for a stream.
|
||||
#[inline]
|
||||
pub fn mix_symph_indiv(
|
||||
// shared buffer to mix into.
|
||||
symph_mix: &mut AudioBuffer<f32>,
|
||||
// buffer to hold built up packet
|
||||
resample_scratch: &mut AudioBuffer<f32>,
|
||||
// the input stream to use
|
||||
input: &mut Parsed,
|
||||
// resampler state and positions into partially read packets
|
||||
local_state: &mut DecodeState,
|
||||
// volume of this source
|
||||
volume: f32,
|
||||
// window into the output UDP buffer to copy opus frames into.
|
||||
// This is set to `Some` IF passthrough is possible (i.e., one live source).
|
||||
mut opus_slot: Option<&mut [u8]>,
|
||||
) -> (MixType, MixStatus) {
|
||||
let mut samples_written = 0;
|
||||
let mut resample_in_progress = false;
|
||||
let mut track_status = MixStatus::Live;
|
||||
let codec_type = input.decoder.codec_params().codec;
|
||||
|
||||
resample_scratch.clear();
|
||||
|
||||
while samples_written != MONO_FRAME_SIZE {
|
||||
// fetch a packet: either in progress, passthrough (early exit), or
|
||||
let source_packet = if local_state.inner_pos != 0 {
|
||||
Some(input.decoder.last_decoded())
|
||||
} else if let Ok(pkt) = input.format.next_packet() {
|
||||
if pkt.track_id() != input.track_id {
|
||||
continue;
|
||||
}
|
||||
|
||||
let buf = pkt.buf();
|
||||
|
||||
// Opus packet passthrough special case.
|
||||
if codec_type == CODEC_TYPE_OPUS && local_state.passthrough != Passthrough::Block {
|
||||
if let Some(slot) = opus_slot.as_mut() {
|
||||
let sample_ct = if buf.is_empty() || buf.len() > i32::MAX as usize {
|
||||
None
|
||||
} else {
|
||||
Some(opus2::packet::get_nb_samples(buf, SAMPLE_RATE))
|
||||
};
|
||||
|
||||
// We don't actually block passthrough until a few violations are
|
||||
// seen. The main one is that most Opus tracks end on a sub-20ms
|
||||
// frame, particularly on Youtube.
|
||||
// However, a frame that's bigger than the target buffer is an instant block.
|
||||
let buf_size_fatal = buf.len() >= slot.len();
|
||||
|
||||
if match sample_ct {
|
||||
Some(Ok(MONO_FRAME_SIZE)) => true,
|
||||
_ => !local_state.record_and_check_passthrough_strike_final(buf_size_fatal),
|
||||
} {
|
||||
slot.write_all(buf)
|
||||
.expect("Bounds check performed, and failure will block passthrough.");
|
||||
|
||||
return (MixType::Passthrough(buf.len()), MixStatus::Live);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
input
|
||||
.decoder
|
||||
.decode(&pkt)
|
||||
.map_err(|e| {
|
||||
track_status = e.into();
|
||||
})
|
||||
.ok()
|
||||
} else {
|
||||
track_status = MixStatus::Ended;
|
||||
None
|
||||
};
|
||||
|
||||
// Cleanup: failed to get the next packet, but still have to convert and mix scratch.
|
||||
if source_packet.is_none() {
|
||||
if resample_in_progress {
|
||||
// fill up remainder of buf with zeroes, resample, mix
|
||||
let (chan_c, resampler, rs_out_buf) = local_state.resampler.as_mut().unwrap();
|
||||
let in_len = resample_scratch.frames();
|
||||
let to_render = resampler.input_frames_next().saturating_sub(in_len);
|
||||
|
||||
if to_render != 0 {
|
||||
resample_scratch.render_reserved(Some(to_render));
|
||||
for plane in resample_scratch.planes_mut().planes() {
|
||||
for val in &mut plane[in_len..] {
|
||||
*val = 0.0f32;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Luckily, we make use of the WHOLE input buffer here.
|
||||
resampler
|
||||
.process_into_buffer(
|
||||
&resample_scratch.planes().planes()[..*chan_c],
|
||||
rs_out_buf,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Calculate true end position using sample rate math
|
||||
let ratio = (rs_out_buf[0].len() as f32) / (resample_scratch.frames() as f32);
|
||||
let out_samples = (ratio * (in_len as f32)).round() as usize;
|
||||
|
||||
mix_resampled(rs_out_buf, symph_mix, samples_written, volume);
|
||||
|
||||
samples_written += out_samples;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
let source_packet = source_packet.unwrap();
|
||||
|
||||
let in_rate = source_packet.spec().rate;
|
||||
let pkt_frames = source_packet.frames();
|
||||
|
||||
if pkt_frames == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
if in_rate == SAMPLE_RATE_RAW as u32 {
|
||||
// No need to resample: mix as standard.
|
||||
let samples_marched = mix_over_ref(
|
||||
&source_packet,
|
||||
symph_mix,
|
||||
local_state.inner_pos,
|
||||
samples_written,
|
||||
volume,
|
||||
);
|
||||
|
||||
samples_written += samples_marched;
|
||||
|
||||
local_state.inner_pos += samples_marched;
|
||||
local_state.inner_pos %= pkt_frames;
|
||||
} else {
|
||||
// NOTE: this should NEVER change in one stream.
|
||||
let chan_c = source_packet.spec().channels.count();
|
||||
let (_, resampler, rs_out_buf) = local_state.resampler.get_or_insert_with(|| {
|
||||
// TODO: integ. error handling here.
|
||||
let resampler = FftFixedOut::new(
|
||||
in_rate as usize,
|
||||
SAMPLE_RATE_RAW,
|
||||
RESAMPLE_OUTPUT_FRAME_SIZE,
|
||||
4,
|
||||
chan_c,
|
||||
)
|
||||
.expect("Failed to create resampler.");
|
||||
let out_buf = resampler.output_buffer_allocate(true);
|
||||
|
||||
(chan_c, resampler, out_buf)
|
||||
});
|
||||
|
||||
let inner_pos = local_state.inner_pos;
|
||||
|
||||
let needed_in_frames = resampler.input_frames_next();
|
||||
let available_frames = pkt_frames - inner_pos;
|
||||
|
||||
let force_copy = resample_in_progress || needed_in_frames > available_frames;
|
||||
if (!force_copy) && matches!(source_packet, AudioBufferRef::F32(_)) {
|
||||
// This is the only case where we can pull off a straight resample...
|
||||
// I would really like if this could be a slice of slices,
|
||||
// but the technology just isn't there yet. And I don't feel like
|
||||
// writing unsafe transformations to do so.
|
||||
|
||||
// NOTE: if let needed as if-let && {bool} is nightly only.
|
||||
if let AudioBufferRef::F32(s_pkt) = source_packet {
|
||||
let refs: Vec<&[f32]> = s_pkt
|
||||
.planes()
|
||||
.planes()
|
||||
.iter()
|
||||
.map(|s| &s[inner_pos..][..needed_in_frames])
|
||||
.collect();
|
||||
|
||||
local_state.inner_pos += needed_in_frames;
|
||||
local_state.inner_pos %= pkt_frames;
|
||||
|
||||
resampler
|
||||
.process_into_buffer(&refs, rs_out_buf, None)
|
||||
.unwrap();
|
||||
} else {
|
||||
unreachable!()
|
||||
}
|
||||
} else {
|
||||
// We either lack enough samples, or have the wrong data format, forcing
|
||||
// a conversion/copy into the buffer.
|
||||
let old_scratch_len = resample_scratch.frames();
|
||||
let missing_frames = needed_in_frames - old_scratch_len;
|
||||
let frames_to_take = available_frames.min(missing_frames);
|
||||
|
||||
resample_scratch.render_reserved(Some(frames_to_take));
|
||||
copy_into_resampler(
|
||||
&source_packet,
|
||||
resample_scratch,
|
||||
inner_pos,
|
||||
old_scratch_len,
|
||||
frames_to_take,
|
||||
);
|
||||
|
||||
local_state.inner_pos += frames_to_take;
|
||||
local_state.inner_pos %= pkt_frames;
|
||||
|
||||
if resample_scratch.frames() == needed_in_frames {
|
||||
resampler
|
||||
.process_into_buffer(
|
||||
&resample_scratch.planes().planes()[..chan_c],
|
||||
rs_out_buf,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
resample_scratch.clear();
|
||||
resample_in_progress = false;
|
||||
} else {
|
||||
// Not enough data to fill the resampler: fetch more.
|
||||
resample_in_progress = true;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let samples_marched = mix_resampled(rs_out_buf, symph_mix, samples_written, volume);
|
||||
|
||||
samples_written += samples_marched;
|
||||
}
|
||||
}
|
||||
|
||||
(MixType::MixedPcm(samples_written), track_status)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn mix_over_ref(
|
||||
source: &AudioBufferRef<'_>,
|
||||
target: &mut AudioBuffer<f32>,
|
||||
source_pos: usize,
|
||||
dest_pos: usize,
|
||||
volume: f32,
|
||||
) -> usize {
|
||||
match source {
|
||||
AudioBufferRef::U8(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
|
||||
AudioBufferRef::U16(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
|
||||
AudioBufferRef::U24(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
|
||||
AudioBufferRef::U32(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
|
||||
AudioBufferRef::S8(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
|
||||
AudioBufferRef::S16(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
|
||||
AudioBufferRef::S24(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
|
||||
AudioBufferRef::S32(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
|
||||
AudioBufferRef::F32(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
|
||||
AudioBufferRef::F64(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn mix_symph_buffer<S>(
|
||||
source: &AudioBuffer<S>,
|
||||
target: &mut AudioBuffer<f32>,
|
||||
source_pos: usize,
|
||||
dest_pos: usize,
|
||||
volume: f32,
|
||||
) -> usize
|
||||
where
|
||||
S: Sample + IntoSample<f32>,
|
||||
{
|
||||
// mix in source_packet[inner_pos..] til end of EITHER buffer.
|
||||
let src_usable = source.frames() - source_pos;
|
||||
let tgt_usable = target.frames() - dest_pos;
|
||||
|
||||
let mix_ct = src_usable.min(tgt_usable);
|
||||
|
||||
let target_chans = target.spec().channels.count();
|
||||
let target_mono = target_chans == 1;
|
||||
let source_chans = source.spec().channels.count();
|
||||
let source_mono = source_chans == 1;
|
||||
|
||||
let source_planes = source.planes();
|
||||
let source_raw_planes = source_planes.planes();
|
||||
|
||||
if source_mono {
|
||||
// mix this signal into *all* output channels at req'd volume.
|
||||
let source_plane = source_raw_planes[0];
|
||||
for d_plane in &mut (*target.planes_mut().planes()) {
|
||||
for (d, s) in d_plane[dest_pos..dest_pos + mix_ct]
|
||||
.iter_mut()
|
||||
.zip(source_plane[source_pos..source_pos + mix_ct].iter())
|
||||
{
|
||||
*d += volume * (*s).into_sample();
|
||||
}
|
||||
}
|
||||
} else if target_mono {
|
||||
// mix all signals into the one target channel: reduce aggregate volume
|
||||
// by n_channels.
|
||||
let vol_adj = 1.0 / (source_chans as f32);
|
||||
let mut t_planes = target.planes_mut();
|
||||
let d_plane = &mut *t_planes.planes()[0];
|
||||
for s_plane in source_raw_planes {
|
||||
for (d, s) in d_plane[dest_pos..dest_pos + mix_ct]
|
||||
.iter_mut()
|
||||
.zip(s_plane[source_pos..source_pos + mix_ct].iter())
|
||||
{
|
||||
*d += volume * vol_adj * (*s).into_sample();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// stereo -> stereo: don't change volume, map input -> output channels w/ no duplication
|
||||
for (d_plane, s_plane) in (*target.planes_mut().planes())
|
||||
.iter_mut()
|
||||
.zip(source_raw_planes[..].iter())
|
||||
{
|
||||
for (d, s) in d_plane[dest_pos..dest_pos + mix_ct]
|
||||
.iter_mut()
|
||||
.zip(s_plane[source_pos..source_pos + mix_ct].iter())
|
||||
{
|
||||
*d += volume * (*s).into_sample();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mix_ct
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn mix_resampled(
|
||||
source: &[Vec<f32>],
|
||||
target: &mut AudioBuffer<f32>,
|
||||
dest_pos: usize,
|
||||
volume: f32,
|
||||
) -> usize {
|
||||
let mix_ct = source[0].len();
|
||||
|
||||
let target_chans = target.spec().channels.count();
|
||||
let target_mono = target_chans == 1;
|
||||
let source_chans = source.len();
|
||||
let source_mono = source_chans == 1;
|
||||
|
||||
// see `mix_symph_buffer` for explanations of stereo<->mono logic.
|
||||
if source_mono {
|
||||
let source_plane = &source[0];
|
||||
for d_plane in &mut (*target.planes_mut().planes()) {
|
||||
for (d, s) in d_plane[dest_pos..dest_pos + mix_ct]
|
||||
.iter_mut()
|
||||
.zip(source_plane)
|
||||
{
|
||||
*d += volume * s;
|
||||
}
|
||||
}
|
||||
} else if target_mono {
|
||||
let vol_adj = 1.0 / (source_chans as f32);
|
||||
let mut t_planes = target.planes_mut();
|
||||
let d_plane = &mut *t_planes.planes()[0];
|
||||
for s_plane in source {
|
||||
for (d, s) in d_plane[dest_pos..dest_pos + mix_ct].iter_mut().zip(s_plane) {
|
||||
*d += volume * vol_adj * s;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (d_plane, s_plane) in (*target.planes_mut().planes())
|
||||
.iter_mut()
|
||||
.zip(source[..].iter())
|
||||
{
|
||||
for (d, s) in d_plane[dest_pos..dest_pos + mix_ct].iter_mut().zip(s_plane) {
|
||||
*d += volume * (*s);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mix_ct
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn copy_into_resampler(
|
||||
source: &AudioBufferRef<'_>,
|
||||
target: &mut AudioBuffer<f32>,
|
||||
source_pos: usize,
|
||||
dest_pos: usize,
|
||||
len: usize,
|
||||
) -> usize {
|
||||
match source {
|
||||
AudioBufferRef::U8(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
|
||||
AudioBufferRef::U16(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
|
||||
AudioBufferRef::U24(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
|
||||
AudioBufferRef::U32(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
|
||||
AudioBufferRef::S8(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
|
||||
AudioBufferRef::S16(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
|
||||
AudioBufferRef::S24(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
|
||||
AudioBufferRef::S32(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
|
||||
AudioBufferRef::F32(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
|
||||
AudioBufferRef::F64(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn copy_symph_buffer<S>(
|
||||
source: &AudioBuffer<S>,
|
||||
target: &mut AudioBuffer<f32>,
|
||||
source_pos: usize,
|
||||
dest_pos: usize,
|
||||
len: usize,
|
||||
) -> usize
|
||||
where
|
||||
S: Sample + IntoSample<f32>,
|
||||
{
|
||||
for (d_plane, s_plane) in (*target.planes_mut().planes())
|
||||
.iter_mut()
|
||||
.zip(source.planes().planes()[..].iter())
|
||||
{
|
||||
for (d, s) in d_plane[dest_pos..dest_pos + len]
|
||||
.iter_mut()
|
||||
.zip(s_plane[source_pos..source_pos + len].iter())
|
||||
{
|
||||
*d = (*s).into_sample();
|
||||
}
|
||||
}
|
||||
|
||||
len
|
||||
}
|
||||
884
app/vendor/songbird/src/driver/tasks/mixer/mod.rs
vendored
884
app/vendor/songbird/src/driver/tasks/mixer/mod.rs
vendored
@@ -1,884 +0,0 @@
|
||||
pub mod mix_logic;
|
||||
mod pool;
|
||||
mod result;
|
||||
pub mod state;
|
||||
pub mod track;
|
||||
mod util;
|
||||
|
||||
use pool::*;
|
||||
use result::*;
|
||||
use state::*;
|
||||
pub use track::*;
|
||||
|
||||
use super::{
|
||||
disposal::DisposalThread,
|
||||
error::{Error, Result},
|
||||
message::*,
|
||||
};
|
||||
use crate::{
|
||||
constants::*,
|
||||
driver::{CryptoMode, MixMode},
|
||||
events::EventStore,
|
||||
input::{Input, Parsed},
|
||||
tracks::{Action, LoopState, PlayError, PlayMode, TrackCommand, TrackHandle, TrackState, View},
|
||||
Config,
|
||||
};
|
||||
use discortp::{
|
||||
discord::MutableKeepalivePacket,
|
||||
rtp::{MutableRtpPacket, RtpPacket},
|
||||
MutablePacket,
|
||||
};
|
||||
use flume::{Receiver, SendError, Sender, TryRecvError};
|
||||
use opus2::{Application as CodingMode, Bitrate, Encoder as OpusEncoder, SoftClip};
|
||||
use rand::random;
|
||||
use rubato::{FftFixedOut, Resampler};
|
||||
use std::{
|
||||
io::Write,
|
||||
result::Result as StdResult,
|
||||
sync::{atomic::Ordering, Arc},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use symphonia_core::{
|
||||
audio::{AudioBuffer, AudioBufferRef, Layout, SampleBuffer, Signal, SignalSpec},
|
||||
codecs::CODEC_TYPE_OPUS,
|
||||
conv::IntoSample,
|
||||
formats::SeekTo,
|
||||
sample::Sample,
|
||||
units::Time,
|
||||
};
|
||||
use tokio::runtime::Handle;
|
||||
use tracing::error;
|
||||
|
||||
#[cfg(test)]
|
||||
use crate::driver::test_config::{OutputMessage, OutputMode};
|
||||
#[cfg(test)]
|
||||
use discortp::Packet as _;
|
||||
|
||||
pub struct Mixer {
|
||||
pub bitrate: Bitrate,
|
||||
pub config: Arc<Config>,
|
||||
pub conn_active: Option<MixerConnection>,
|
||||
pub deadline: Instant,
|
||||
pub disposer: DisposalThread,
|
||||
pub encoder: OpusEncoder,
|
||||
pub interconnect: Interconnect,
|
||||
pub mix_rx: Receiver<MixerMessage>,
|
||||
pub muted: bool,
|
||||
// pub packet: [u8; VOICE_PACKET_MAX],
|
||||
pub prevent_events: bool,
|
||||
pub silence_frames: u8,
|
||||
pub soft_clip: SoftClip,
|
||||
thread_pool: BlockyTaskPool,
|
||||
pub ws: Option<Sender<WsMessage>>,
|
||||
|
||||
pub keepalive_deadline: Instant,
|
||||
pub keepalive_packet: [u8; MutableKeepalivePacket::minimum_packet_size()],
|
||||
|
||||
pub tracks: Vec<InternalTrack>,
|
||||
track_handles: Vec<TrackHandle>,
|
||||
|
||||
sample_buffer: SampleBuffer<f32>,
|
||||
symph_mix: AudioBuffer<f32>,
|
||||
resample_scratch: AudioBuffer<f32>,
|
||||
|
||||
#[cfg(test)]
|
||||
pub remaining_loops: Option<u64>,
|
||||
|
||||
#[cfg(test)]
|
||||
raw_msg: Option<OutputMessage>,
|
||||
}
|
||||
|
||||
fn new_encoder(bitrate: Bitrate, mix_mode: MixMode) -> Result<OpusEncoder> {
|
||||
let mut encoder = OpusEncoder::new(SAMPLE_RATE, mix_mode.to_opus(), CodingMode::Audio)?;
|
||||
encoder.set_bitrate(bitrate)?;
|
||||
|
||||
Ok(encoder)
|
||||
}
|
||||
|
||||
impl Mixer {
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
mix_rx: Receiver<MixerMessage>,
|
||||
async_handle: Handle,
|
||||
interconnect: Interconnect,
|
||||
config: Config,
|
||||
) -> Self {
|
||||
let bitrate = DEFAULT_BITRATE;
|
||||
let encoder = new_encoder(bitrate, config.mix_mode)
|
||||
.expect("Failed to create encoder in mixing thread with known-good values.");
|
||||
let soft_clip = SoftClip::new(config.mix_mode.to_opus());
|
||||
|
||||
let keepalive_packet = [0u8; MutableKeepalivePacket::minimum_packet_size()];
|
||||
|
||||
let tracks = Vec::with_capacity(usize::from(1.max(config.preallocated_tracks)));
|
||||
let track_handles = Vec::with_capacity(usize::from(1.max(config.preallocated_tracks)));
|
||||
|
||||
let thread_pool = BlockyTaskPool::new(async_handle);
|
||||
|
||||
let symph_layout = config.mix_mode.symph_layout();
|
||||
|
||||
let disposer = config.disposer.clone().unwrap_or_default();
|
||||
let config = config.into();
|
||||
|
||||
let sample_buffer = SampleBuffer::<f32>::new(
|
||||
MONO_FRAME_SIZE as u64,
|
||||
symphonia_core::audio::SignalSpec::new_with_layout(
|
||||
SAMPLE_RATE_RAW as u32,
|
||||
symph_layout,
|
||||
),
|
||||
);
|
||||
let symph_mix = AudioBuffer::<f32>::new(
|
||||
MONO_FRAME_SIZE as u64,
|
||||
symphonia_core::audio::SignalSpec::new_with_layout(
|
||||
SAMPLE_RATE_RAW as u32,
|
||||
symph_layout,
|
||||
),
|
||||
);
|
||||
let resample_scratch = AudioBuffer::<f32>::new(
|
||||
MONO_FRAME_SIZE as u64,
|
||||
SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, Layout::Stereo),
|
||||
);
|
||||
|
||||
let deadline = Instant::now();
|
||||
|
||||
Self {
|
||||
bitrate,
|
||||
config,
|
||||
conn_active: None,
|
||||
deadline,
|
||||
disposer,
|
||||
encoder,
|
||||
interconnect,
|
||||
mix_rx,
|
||||
muted: false,
|
||||
prevent_events: false,
|
||||
silence_frames: 0,
|
||||
soft_clip,
|
||||
thread_pool,
|
||||
ws: None,
|
||||
|
||||
keepalive_deadline: deadline,
|
||||
keepalive_packet,
|
||||
|
||||
tracks,
|
||||
track_handles,
|
||||
|
||||
sample_buffer,
|
||||
symph_mix,
|
||||
resample_scratch,
|
||||
|
||||
#[cfg(test)]
|
||||
remaining_loops: None,
|
||||
#[cfg(test)]
|
||||
raw_msg: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn set_bitrate(&mut self, bitrate: Bitrate) -> Result<()> {
|
||||
self.encoder.set_bitrate(bitrate).map_err(Into::into)
|
||||
}
|
||||
|
||||
pub(crate) fn do_rebuilds(
|
||||
&mut self,
|
||||
event_failure: bool,
|
||||
conn_failure: bool,
|
||||
) -> StdResult<(), SendError<CoreMessage>> {
|
||||
// event failure? rebuild interconnect.
|
||||
// ws or udp failure? full connect
|
||||
// (soft reconnect is covered by the ws task.)
|
||||
//
|
||||
// in both cases, send failure is fatal,
|
||||
// but will only occur on disconnect.
|
||||
if event_failure {
|
||||
self.rebuild_interconnect()?;
|
||||
}
|
||||
|
||||
if conn_failure {
|
||||
self.full_reconnect_gateway()?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn rebuild_interconnect(&mut self) -> StdResult<(), SendError<CoreMessage>> {
|
||||
self.prevent_events = true;
|
||||
self.interconnect
|
||||
.core
|
||||
.send(CoreMessage::RebuildInterconnect)
|
||||
}
|
||||
|
||||
pub(crate) fn full_reconnect_gateway(&mut self) -> StdResult<(), SendError<CoreMessage>> {
|
||||
self.conn_active = None;
|
||||
self.interconnect.core.send(CoreMessage::FullReconnect)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn handle_message(
|
||||
&mut self,
|
||||
msg: MixerMessage,
|
||||
packet: &mut [u8],
|
||||
) -> (bool, bool, bool) {
|
||||
let mut events_failure = false;
|
||||
let mut conn_failure = false;
|
||||
let mut should_exit = false;
|
||||
|
||||
let error = match msg {
|
||||
MixerMessage::AddTrack(t) => self.add_track(*t),
|
||||
MixerMessage::SetTrack(t) => {
|
||||
self.tracks.clear();
|
||||
self.track_handles.clear();
|
||||
|
||||
let mut out = self.fire_event(EventMessage::RemoveAllTracks);
|
||||
|
||||
if let Some(t) = t {
|
||||
// Do this unconditionally: this affects local state infallibly,
|
||||
// with the event installation being the remote part.
|
||||
if let Err(e) = self.add_track(*t) {
|
||||
out = Err(e);
|
||||
}
|
||||
}
|
||||
|
||||
out
|
||||
},
|
||||
MixerMessage::SetBitrate(b) => {
|
||||
self.bitrate = b;
|
||||
if let Err(e) = self.set_bitrate(b) {
|
||||
error!("Failed to update bitrate {:?}", e);
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
MixerMessage::SetMute(m) => {
|
||||
self.muted = m;
|
||||
Ok(())
|
||||
},
|
||||
MixerMessage::SetConn(conn, ssrc) => {
|
||||
self.conn_active = Some(conn);
|
||||
let mut rtp = MutableRtpPacket::new(packet).expect(
|
||||
"Too few bytes in self.packet for RTP header.\
|
||||
(Blame: VOICE_PACKET_MAX?)",
|
||||
);
|
||||
rtp.set_ssrc(ssrc);
|
||||
rtp.set_sequence(random::<u16>().into());
|
||||
rtp.set_timestamp(random::<u32>().into());
|
||||
self.deadline = Instant::now();
|
||||
|
||||
self.update_keepalive(ssrc);
|
||||
Ok(())
|
||||
},
|
||||
MixerMessage::DropConn => {
|
||||
self.conn_active = None;
|
||||
Ok(())
|
||||
},
|
||||
MixerMessage::ReplaceInterconnect(i) => {
|
||||
self.prevent_events = false;
|
||||
|
||||
if let Some(ws) = &self.ws {
|
||||
conn_failure |= ws.send(WsMessage::ReplaceInterconnect(i.clone())).is_err();
|
||||
}
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
if let Some(conn) = &self.conn_active {
|
||||
conn_failure |= conn
|
||||
.udp_rx
|
||||
.send(UdpRxMessage::ReplaceInterconnect(i.clone()))
|
||||
.is_err();
|
||||
}
|
||||
|
||||
self.interconnect = i;
|
||||
|
||||
self.rebuild_tracks()
|
||||
},
|
||||
MixerMessage::SetConfig(new_config) => {
|
||||
if new_config.mix_mode != self.config.mix_mode {
|
||||
self.soft_clip = SoftClip::new(new_config.mix_mode.to_opus());
|
||||
if let Ok(enc) = new_encoder(self.bitrate, new_config.mix_mode) {
|
||||
self.encoder = enc;
|
||||
} else {
|
||||
self.bitrate = DEFAULT_BITRATE;
|
||||
self.encoder = new_encoder(self.bitrate, new_config.mix_mode)
|
||||
.expect("Failed fallback rebuild of OpusEncoder with safe inputs.");
|
||||
}
|
||||
|
||||
let sl = new_config.mix_mode.symph_layout();
|
||||
self.sample_buffer = SampleBuffer::<f32>::new(
|
||||
MONO_FRAME_SIZE as u64,
|
||||
SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, sl),
|
||||
);
|
||||
self.symph_mix = AudioBuffer::<f32>::new(
|
||||
MONO_FRAME_SIZE as u64,
|
||||
SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, sl),
|
||||
);
|
||||
}
|
||||
|
||||
self.config = Arc::new(
|
||||
#[cfg(feature = "receive")]
|
||||
new_config.clone(),
|
||||
#[cfg(not(feature = "receive"))]
|
||||
new_config,
|
||||
);
|
||||
|
||||
let preallocated_tracks = usize::from(self.config.preallocated_tracks);
|
||||
if self.tracks.capacity() < preallocated_tracks {
|
||||
self.tracks.reserve(preallocated_tracks - self.tracks.len());
|
||||
}
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
if let Some(conn) = &self.conn_active {
|
||||
if let crate::driver::DecodeMode::Decode(decode_config) = new_config.decode_mode
|
||||
{
|
||||
let msg = UdpRxMessage::SetConfig(decode_config);
|
||||
conn_failure |= conn.udp_rx.send(msg).is_err();
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
},
|
||||
MixerMessage::RebuildEncoder => match new_encoder(self.bitrate, self.config.mix_mode) {
|
||||
Ok(encoder) => {
|
||||
self.encoder = encoder;
|
||||
Ok(())
|
||||
},
|
||||
Err(e) => {
|
||||
error!("Failed to rebuild encoder. Resetting bitrate. {:?}", e);
|
||||
self.bitrate = DEFAULT_BITRATE;
|
||||
self.encoder = new_encoder(self.bitrate, self.config.mix_mode)
|
||||
.expect("Failed fallback rebuild of OpusEncoder with safe inputs.");
|
||||
Ok(())
|
||||
},
|
||||
},
|
||||
MixerMessage::Ws(new_ws_handle) => {
|
||||
self.ws = new_ws_handle;
|
||||
if let Err(e) = self.send_gateway_speaking() {
|
||||
conn_failure |= e.should_trigger_connect();
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
MixerMessage::Poison => {
|
||||
should_exit = true;
|
||||
Ok(())
|
||||
},
|
||||
};
|
||||
|
||||
if let Err(e) = error {
|
||||
events_failure |= e.should_trigger_interconnect_rebuild();
|
||||
conn_failure |= e.should_trigger_connect();
|
||||
}
|
||||
|
||||
(events_failure, conn_failure, should_exit)
|
||||
}
|
||||
|
||||
pub(crate) fn update_keepalive(&mut self, ssrc: u32) {
|
||||
let mut ka = MutableKeepalivePacket::new(&mut self.keepalive_packet[..])
|
||||
.expect("FATAL: Insufficient bytes given to keepalive packet.");
|
||||
ka.set_ssrc(ssrc);
|
||||
self.keepalive_deadline = self.deadline + UDP_KEEPALIVE_GAP;
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn fire_event(&self, event: EventMessage) -> Result<()> {
|
||||
// As this task is responsible for noticing the potential death of an event context,
|
||||
// it's responsible for not forcibly recreating said context repeatedly.
|
||||
if !self.prevent_events {
|
||||
self.interconnect.events.send(event)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn add_track(&mut self, track: TrackContext) -> Result<()> {
|
||||
let (track, evts, state, handle) = InternalTrack::decompose_track(track);
|
||||
self.tracks.push(track);
|
||||
self.track_handles.push(handle.clone());
|
||||
self.interconnect
|
||||
.events
|
||||
.send(EventMessage::AddTrack(evts, state, handle))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// rebuilds the event thread's view of each track, in event of a full rebuild.
|
||||
#[inline]
|
||||
fn rebuild_tracks(&mut self) -> Result<()> {
|
||||
for (track, handle) in self.tracks.iter().zip(self.track_handles.iter()) {
|
||||
let evts = EventStore::default();
|
||||
let state = track.state();
|
||||
let handle = handle.clone();
|
||||
|
||||
self.interconnect
|
||||
.events
|
||||
.send(EventMessage::AddTrack(evts, state, handle))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn audio_commands_events(&mut self) -> Result<()> {
|
||||
// Apply user commands.
|
||||
for (i, track) in self.tracks.iter_mut().enumerate() {
|
||||
// This causes fallible event system changes,
|
||||
// but if the event thread has died then we'll certainly
|
||||
// detect that on the tick later.
|
||||
// Changes to play state etc. MUST all be handled.
|
||||
let action = track.process_commands(i, &self.interconnect);
|
||||
|
||||
if let Some(req) = action.seek_point {
|
||||
track.seek(
|
||||
i,
|
||||
req,
|
||||
&self.interconnect,
|
||||
&self.thread_pool,
|
||||
&self.config,
|
||||
self.prevent_events,
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(callback) = action.make_playable {
|
||||
if let Err(e) = track.get_or_ready_input(
|
||||
i,
|
||||
&self.interconnect,
|
||||
&self.thread_pool,
|
||||
&self.config,
|
||||
self.prevent_events,
|
||||
) {
|
||||
track.callbacks.make_playable = Some(callback);
|
||||
if let Some(fail) = e.as_user() {
|
||||
track.playing = PlayMode::Errored(fail);
|
||||
}
|
||||
if let Some(req) = e.into_seek_request() {
|
||||
track.seek(
|
||||
i,
|
||||
req,
|
||||
&self.interconnect,
|
||||
&self.thread_pool,
|
||||
&self.config,
|
||||
self.prevent_events,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// Track is already ready: don't register callback and just act.
|
||||
drop(callback.send(Ok(())));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut i = 0;
|
||||
while i < self.tracks.len() {
|
||||
let track = self
|
||||
.tracks
|
||||
.get_mut(i)
|
||||
.expect("Tried to remove an illegal track index.");
|
||||
|
||||
if track.playing.is_done() {
|
||||
let p_state = track.playing.clone();
|
||||
let to_drop = self.tracks.swap_remove(i);
|
||||
self.disposer
|
||||
.dispose(DisposalMessage::Track(Box::new(to_drop)));
|
||||
|
||||
let to_drop = self.track_handles.swap_remove(i);
|
||||
self.disposer.dispose(DisposalMessage::Handle(to_drop));
|
||||
|
||||
self.fire_event(EventMessage::ChangeState(
|
||||
i,
|
||||
TrackStateChange::Mode(p_state),
|
||||
))?;
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Tick -- receive side also handles removals in same manner after it increments
|
||||
// times etc.
|
||||
self.fire_event(EventMessage::Tick)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[inline]
|
||||
pub(crate) fn test_signal_empty_tick(&self) {
|
||||
match &self.config.override_connection {
|
||||
Some(OutputMode::Raw(tx)) =>
|
||||
drop(tx.send(crate::driver::test_config::TickMessage::NoEl)),
|
||||
Some(OutputMode::Rtp(tx)) =>
|
||||
drop(tx.send(crate::driver::test_config::TickMessage::NoEl)),
|
||||
None => {},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn crypto_mode(&self) -> CryptoMode {
|
||||
let mode = self.conn_active.as_ref().map(|v| v.crypto_state.kind());
|
||||
if cfg!(not(test)) {
|
||||
mode.expect("Shouldn't be mixing packets without access to a cipher + UDP dest.")
|
||||
} else {
|
||||
mode.unwrap_or_else(|| self.config.crypto_mode)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn mix_and_build_packet(&mut self, packet: &mut [u8]) -> Result<usize> {
|
||||
// symph_mix is an `AudioBuffer` (planar format), we need to convert this
|
||||
// later into an interleaved `SampleBuffer` for libopus.
|
||||
self.symph_mix.clear();
|
||||
self.symph_mix.render_reserved(Some(MONO_FRAME_SIZE));
|
||||
self.resample_scratch.clear();
|
||||
|
||||
// Walk over all the audio files, combining into one audio frame according
|
||||
// to volume, play state, etc.
|
||||
let mut mix_len = {
|
||||
let out = self.mix_tracks(packet);
|
||||
|
||||
self.sample_buffer.copy_interleaved_typed(&self.symph_mix);
|
||||
|
||||
out
|
||||
};
|
||||
|
||||
if self.muted {
|
||||
mix_len = MixType::MixedPcm(0);
|
||||
}
|
||||
|
||||
// Explicit "Silence" frame handling: if there is no mixed data, we must send
|
||||
// ~5 frames of silence (unless another good audio frame appears) before we
|
||||
// stop sending RTP frames.
|
||||
if mix_len == MixType::MixedPcm(0) {
|
||||
if self.silence_frames > 0 {
|
||||
self.silence_frames -= 1;
|
||||
let mut rtp = MutableRtpPacket::new(packet).expect(
|
||||
"FATAL: Too few bytes in self.packet for RTP header.\
|
||||
(Blame: VOICE_PACKET_MAX?)",
|
||||
);
|
||||
|
||||
let payload = rtp.payload_mut();
|
||||
let pre_len = self.crypto_mode().payload_prefix_len();
|
||||
|
||||
payload[pre_len..pre_len + SILENT_FRAME.len()].copy_from_slice(&SILENT_FRAME[..]);
|
||||
|
||||
mix_len = MixType::Passthrough(SILENT_FRAME.len());
|
||||
} else {
|
||||
// Per official guidelines, send 5x silence BEFORE we stop speaking.
|
||||
return Ok(0);
|
||||
}
|
||||
} else {
|
||||
self.silence_frames = 5;
|
||||
|
||||
if let MixType::MixedPcm(n) = mix_len {
|
||||
if self.config.use_softclip {
|
||||
self.soft_clip.apply(
|
||||
&mut self.sample_buffer.samples_mut()
|
||||
[..n * self.config.mix_mode.channels()],
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// For the benefit of test cases, send the raw un-RTP'd data.
|
||||
#[cfg(test)]
|
||||
let out = if let Some(OutputMode::Raw(_)) = &self.config.override_connection {
|
||||
// This case has been handled before buffer clearing above.
|
||||
let msg = match mix_len {
|
||||
MixType::Passthrough(len) if len == SILENT_FRAME.len() => OutputMessage::Silent,
|
||||
MixType::Passthrough(len) => {
|
||||
let rtp = RtpPacket::new(packet).expect(
|
||||
"FATAL: Too few bytes in self.packet for RTP header.\
|
||||
(Blame: VOICE_PACKET_MAX?)",
|
||||
);
|
||||
let payload = rtp.payload();
|
||||
let opus_frame =
|
||||
(payload[self.crypto_mode().payload_prefix_len()..][..len]).to_vec();
|
||||
|
||||
OutputMessage::Passthrough(opus_frame)
|
||||
},
|
||||
MixType::MixedPcm(_) => OutputMessage::Mixed(
|
||||
self.sample_buffer.samples()[..self.config.mix_mode.sample_count_in_frame()]
|
||||
.to_vec(),
|
||||
),
|
||||
};
|
||||
|
||||
self.raw_msg = Some(msg);
|
||||
|
||||
Ok(1)
|
||||
} else {
|
||||
self.prep_packet(mix_len, packet)
|
||||
};
|
||||
|
||||
#[cfg(not(test))]
|
||||
let out = self.prep_packet(mix_len, packet);
|
||||
|
||||
// Zero out all planes of the mix buffer if any audio was written.
|
||||
if matches!(mix_len, MixType::MixedPcm(a) if a > 0) {
|
||||
for plane in self.symph_mix.planes_mut().planes() {
|
||||
plane.fill(0.0);
|
||||
}
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn prep_packet(&mut self, mix_len: MixType, packet: &mut [u8]) -> Result<usize> {
|
||||
let send_buffer = self.sample_buffer.samples();
|
||||
|
||||
let conn = self
|
||||
.conn_active
|
||||
.as_mut()
|
||||
.expect("Shouldn't be mixing packets without access to a cipher + UDP dest.");
|
||||
|
||||
let mut rtp = MutableRtpPacket::new(packet).expect(
|
||||
"FATAL: Too few bytes in self.packet for RTP header.\
|
||||
(Blame: VOICE_PACKET_MAX?)",
|
||||
);
|
||||
|
||||
let payload = rtp.payload_mut();
|
||||
let crypto_mode = conn.crypto_state.kind();
|
||||
let first_payload_byte = crypto_mode.payload_prefix_len();
|
||||
let total_payload_space = payload.len() - crypto_mode.payload_suffix_len();
|
||||
|
||||
// If passthrough, Opus payload in place already.
|
||||
// Else encode into buffer with space for AEAD encryption headers.
|
||||
let mut payload_len = match mix_len {
|
||||
MixType::Passthrough(opus_len) => opus_len,
|
||||
MixType::MixedPcm(_samples) => self.encoder.encode_float(
|
||||
&send_buffer[..self.config.mix_mode.sample_count_in_frame()],
|
||||
&mut payload[first_payload_byte..total_payload_space],
|
||||
)?,
|
||||
};
|
||||
|
||||
if conn.dave_protocol_version.load(Ordering::Relaxed) != 0 {
|
||||
if let Some(ref mut dave_session) = *conn.dave_session.write().unwrap() {
|
||||
if dave_session.is_ready() {
|
||||
let encrypted = dave_session
|
||||
.encrypt_opus(
|
||||
&payload[first_payload_byte..first_payload_byte + payload_len],
|
||||
)?
|
||||
.into_owned();
|
||||
payload_len = encrypted.len();
|
||||
|
||||
payload[first_payload_byte..first_payload_byte + payload_len]
|
||||
.copy_from_slice(&encrypted);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let final_payload_size = conn
|
||||
.crypto_state
|
||||
.write_packet_nonce(&mut rtp, first_payload_byte + payload_len);
|
||||
|
||||
// Packet encryption ignored in test modes.
|
||||
#[cfg(not(test))]
|
||||
let encrypt = true;
|
||||
#[cfg(test)]
|
||||
let encrypt = self.config.override_connection.is_none();
|
||||
|
||||
if encrypt {
|
||||
conn.cipher
|
||||
.encrypt_pkt_in_place(&mut rtp, final_payload_size)?;
|
||||
}
|
||||
|
||||
Ok(RtpPacket::minimum_packet_size() + final_payload_size)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn send_packet(&self, packet: &[u8]) -> Result<()> {
|
||||
#[cfg(test)]
|
||||
let send_status = if let Some(OutputMode::Raw(tx)) = &self.config.override_connection {
|
||||
// This case has been handled before buffer clearing in `mix_and_build_packet`.
|
||||
drop(tx.send(self.raw_msg.clone().unwrap().into()));
|
||||
|
||||
Ok(())
|
||||
} else {
|
||||
self.send_packet_(packet)
|
||||
};
|
||||
|
||||
#[cfg(not(test))]
|
||||
let send_status = self.send_packet_(packet);
|
||||
|
||||
send_status.or_else(Error::disarm_would_block)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn send_packet_(&self, packet: &[u8]) -> Result<()> {
|
||||
let conn = self
|
||||
.conn_active
|
||||
.as_ref()
|
||||
.expect("Shouldn't be mixing packets without access to a cipher + UDP dest.");
|
||||
|
||||
#[cfg(test)]
|
||||
if let Some(OutputMode::Rtp(tx)) = &self.config.override_connection {
|
||||
// Test mode: send unencrypted (compressed) packets to local receiver.
|
||||
drop(tx.send(packet.to_vec().into()));
|
||||
} else {
|
||||
conn.udp_tx.send(packet)?;
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
// Normal operation: send encrypted payload to UDP Tx task.
|
||||
conn.udp_tx.send(packet)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn check_and_send_keepalive(&mut self, now: Option<Instant>) -> Result<()> {
|
||||
if let Some(conn) = self.conn_active.as_mut() {
|
||||
let now = now.unwrap_or_else(Instant::now);
|
||||
if now >= self.keepalive_deadline {
|
||||
conn.udp_tx.send(&self.keepalive_packet)?;
|
||||
self.keepalive_deadline += UDP_KEEPALIVE_GAP;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn send_gateway_speaking(&self) -> Result<()> {
|
||||
if let Some(ws) = &self.ws {
|
||||
ws.send(WsMessage::Speaking(true))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn send_gateway_not_speaking(&self) {
|
||||
if let Some(ws) = &self.ws {
|
||||
// NOTE: this explicit `drop` should prevent a catastrophic thread pileup.
|
||||
// A full reconnect might cause an inner closed connection.
|
||||
// It's safer to leave the central task to clean this up and
|
||||
// pass the mixer a new channel.
|
||||
drop(ws.send(WsMessage::Speaking(false)));
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn mix_tracks(&mut self, packet: &mut [u8]) -> MixType {
|
||||
// Get a slice of bytes to write in data for Opus packet passthrough.
|
||||
let mut rtp = MutableRtpPacket::new(packet).expect(
|
||||
"FATAL: Too few bytes in self.packet for RTP header.\
|
||||
(Blame: VOICE_PACKET_MAX?)",
|
||||
);
|
||||
let payload = rtp.payload_mut();
|
||||
let opus_frame = &mut payload[self.crypto_mode().payload_prefix_len()..];
|
||||
|
||||
// Opus frame passthrough.
|
||||
// This requires that we have only one PLAYING track, who has volume 1.0, and an
|
||||
// Opus codec type (verified later in mix_symph_indiv).
|
||||
//
|
||||
// We *could* cache the number of live tracks separately, but that makes this
|
||||
// quite fragile given all the ways a user can alter the PlayMode.
|
||||
let mut num_live = 0;
|
||||
let mut last_live_vol = 1.0;
|
||||
for track in &self.tracks {
|
||||
if track.playing.is_playing() {
|
||||
num_live += 1;
|
||||
last_live_vol = track.volume;
|
||||
}
|
||||
}
|
||||
let do_passthrough = num_live == 1 && (last_live_vol - 1.0).abs() < f32::EPSILON;
|
||||
|
||||
let mut len = 0;
|
||||
for (i, track) in self.tracks.iter_mut().enumerate() {
|
||||
let vol = track.volume;
|
||||
|
||||
// This specifically tries to get tracks who are "preparing",
|
||||
// so that event handlers and the like can all be fired without
|
||||
// the track being in a `Play` state.
|
||||
if !track.should_check_input() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let should_play = track.playing.is_playing();
|
||||
|
||||
let input = track.get_or_ready_input(
|
||||
i,
|
||||
&self.interconnect,
|
||||
&self.thread_pool,
|
||||
&self.config,
|
||||
self.prevent_events,
|
||||
);
|
||||
|
||||
let (input, mix_state) = match input {
|
||||
Ok(i) => i,
|
||||
Err(InputReadyingError::Waiting) => continue,
|
||||
Err(InputReadyingError::NeedsSeek(req)) => {
|
||||
track.seek(
|
||||
i,
|
||||
req,
|
||||
&self.interconnect,
|
||||
&self.thread_pool,
|
||||
&self.config,
|
||||
self.prevent_events,
|
||||
);
|
||||
continue;
|
||||
},
|
||||
// TODO: allow for retry in given time.
|
||||
Err(e) => {
|
||||
if let Some(fail) = e.as_user() {
|
||||
track.playing = PlayMode::Errored(fail);
|
||||
}
|
||||
continue;
|
||||
},
|
||||
};
|
||||
|
||||
// Now that we have dealt with potential errors in preparing tracks,
|
||||
// only do any mixing if the track is to be played!
|
||||
if !should_play {
|
||||
continue;
|
||||
}
|
||||
|
||||
let (mix_type, status) = mix_logic::mix_symph_indiv(
|
||||
&mut self.symph_mix,
|
||||
&mut self.resample_scratch,
|
||||
input,
|
||||
mix_state,
|
||||
vol,
|
||||
do_passthrough.then_some(&mut *opus_frame),
|
||||
);
|
||||
|
||||
let return_here = if let MixType::MixedPcm(pcm_len) = mix_type {
|
||||
len = len.max(pcm_len);
|
||||
false
|
||||
} else {
|
||||
if mix_state.passthrough == Passthrough::Inactive {
|
||||
input.decoder.reset();
|
||||
}
|
||||
mix_state.passthrough = Passthrough::Active;
|
||||
true
|
||||
};
|
||||
|
||||
// FIXME: allow Ended to trigger a seek/loop/revisit in the same mix cycle?
|
||||
// Would this be possible with special-casing to mark some inputs as fast
|
||||
// to recreate? Probably not doable in the general case.
|
||||
match status {
|
||||
MixStatus::Live => track.step_frame(),
|
||||
MixStatus::Errored(e) =>
|
||||
track.playing = PlayMode::Errored(PlayError::Decode(e.into())),
|
||||
MixStatus::Ended if track.do_loop() => {
|
||||
drop(self.track_handles[i].seek(Duration::default()));
|
||||
if !self.prevent_events {
|
||||
// position update is sent out later, when the seek concludes.
|
||||
drop(self.interconnect.events.send(EventMessage::ChangeState(
|
||||
i,
|
||||
TrackStateChange::Loops(track.loops, false),
|
||||
)));
|
||||
}
|
||||
},
|
||||
MixStatus::Ended => {
|
||||
track.end();
|
||||
},
|
||||
}
|
||||
|
||||
// This needs to happen here due to borrow checker shenanigans.
|
||||
if return_here {
|
||||
return mix_type;
|
||||
}
|
||||
}
|
||||
|
||||
MixType::MixedPcm(len)
|
||||
}
|
||||
}
|
||||
141
app/vendor/songbird/src/driver/tasks/mixer/pool.rs
vendored
141
app/vendor/songbird/src/driver/tasks/mixer/pool.rs
vendored
@@ -1,141 +0,0 @@
|
||||
use super::util::copy_seek_to;
|
||||
|
||||
use crate::{
|
||||
driver::tasks::message::MixerInputResultMessage,
|
||||
input::{AudioStream, AudioStreamError, Compose, Input, LiveInput, Parsed},
|
||||
Config,
|
||||
};
|
||||
use flume::Sender;
|
||||
use rusty_pool::ThreadPool;
|
||||
use std::{result::Result as StdResult, sync::Arc, time::Duration};
|
||||
use symphonia_core::{
|
||||
formats::{SeekMode, SeekTo},
|
||||
io::MediaSource,
|
||||
};
|
||||
use tokio::runtime::Handle;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct BlockyTaskPool {
|
||||
pool: ThreadPool,
|
||||
handle: Handle,
|
||||
}
|
||||
|
||||
impl BlockyTaskPool {
|
||||
pub fn new(handle: Handle) -> Self {
|
||||
Self {
|
||||
pool: ThreadPool::new(0, 64, Duration::from_secs(5)),
|
||||
handle,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn create(
|
||||
&self,
|
||||
callback: Sender<MixerInputResultMessage>,
|
||||
input: Input,
|
||||
seek_time: Option<SeekTo>,
|
||||
config: Arc<Config>,
|
||||
) {
|
||||
// Moves an Input from Lazy -> Live.
|
||||
// We either do this on this pool, or move it to the tokio executor as the source requires.
|
||||
// This takes a seek_time to pass on and execute *after* parsing (i.e., back-seek on
|
||||
// read-only stream).
|
||||
match input {
|
||||
Input::Lazy(mut lazy) => {
|
||||
let far_pool = self.clone();
|
||||
if lazy.should_create_async() {
|
||||
self.handle.spawn(async move {
|
||||
let out = lazy.create_async().await;
|
||||
far_pool.send_to_parse(out, lazy, callback, seek_time, config);
|
||||
});
|
||||
} else {
|
||||
self.pool.execute(move || {
|
||||
let out = lazy.create();
|
||||
far_pool.send_to_parse(out, lazy, callback, seek_time, config);
|
||||
});
|
||||
}
|
||||
},
|
||||
Input::Live(live, maybe_create) =>
|
||||
self.parse(config, callback, live, maybe_create, seek_time),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn send_to_parse(
|
||||
&self,
|
||||
create_res: StdResult<AudioStream<Box<dyn MediaSource>>, AudioStreamError>,
|
||||
rec: Box<dyn Compose>,
|
||||
callback: Sender<MixerInputResultMessage>,
|
||||
seek_time: Option<SeekTo>,
|
||||
config: Arc<Config>,
|
||||
) {
|
||||
match create_res {
|
||||
Ok(o) => {
|
||||
self.parse(config, callback, LiveInput::Raw(o), Some(rec), seek_time);
|
||||
},
|
||||
Err(e) => {
|
||||
drop(callback.send(MixerInputResultMessage::CreateErr(e.into())));
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(
|
||||
&self,
|
||||
config: Arc<Config>,
|
||||
callback: Sender<MixerInputResultMessage>,
|
||||
input: LiveInput,
|
||||
rec: Option<Box<dyn Compose>>,
|
||||
seek_time: Option<SeekTo>,
|
||||
) {
|
||||
let pool_clone = self.clone();
|
||||
|
||||
self.pool.execute(move || {
|
||||
match input.promote(config.codec_registry, config.format_registry) {
|
||||
Ok(LiveInput::Parsed(parsed)) => match seek_time {
|
||||
// If seek time is zero, then wipe it out.
|
||||
// Some formats (MKV) make SeekTo(0) require a backseek to realign with the
|
||||
// current page.
|
||||
Some(seek_time) if !super::util::seek_to_is_zero(&seek_time) => {
|
||||
pool_clone.seek(callback, parsed, rec, seek_time, false, config);
|
||||
},
|
||||
_ => {
|
||||
drop(callback.send(MixerInputResultMessage::Built(parsed, rec)));
|
||||
},
|
||||
},
|
||||
Ok(_) => unreachable!(),
|
||||
Err(e) => {
|
||||
drop(callback.send(MixerInputResultMessage::ParseErr(e.into())));
|
||||
},
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub fn seek(
|
||||
&self,
|
||||
callback: Sender<MixerInputResultMessage>,
|
||||
mut input: Parsed,
|
||||
rec: Option<Box<dyn Compose>>,
|
||||
seek_time: SeekTo,
|
||||
// Not all of symphonia's formats bother to return SeekErrorKind::ForwardOnly.
|
||||
// So, we need *this* flag.
|
||||
backseek_needed: bool,
|
||||
config: Arc<Config>,
|
||||
) {
|
||||
let pool_clone = self.clone();
|
||||
|
||||
self.pool.execute(move || match rec {
|
||||
Some(rec) if (!input.supports_backseek) && backseek_needed => {
|
||||
pool_clone.create(callback, Input::Lazy(rec), Some(seek_time), config);
|
||||
},
|
||||
_ => {
|
||||
let seek_result = input
|
||||
.format
|
||||
.seek(SeekMode::Accurate, copy_seek_to(&seek_time));
|
||||
input.decoder.reset();
|
||||
drop(callback.send(MixerInputResultMessage::Seek(
|
||||
input,
|
||||
rec,
|
||||
seek_result.map_err(Arc::new),
|
||||
)));
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
use crate::{
|
||||
input::AudioStreamError,
|
||||
tracks::{PlayError, SeekRequest},
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use symphonia_core::errors::Error as SymphoniaError;
|
||||
|
||||
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
|
||||
pub enum MixType {
|
||||
Passthrough(usize),
|
||||
MixedPcm(usize),
|
||||
}
|
||||
|
||||
pub enum MixStatus {
|
||||
Live,
|
||||
Ended,
|
||||
Errored(SymphoniaError),
|
||||
}
|
||||
|
||||
impl From<SymphoniaError> for MixStatus {
|
||||
fn from(e: SymphoniaError) -> Self {
|
||||
Self::Errored(e)
|
||||
}
|
||||
}
|
||||
|
||||
// The Symph errors are Arc'd here since if they come up, they will always
|
||||
// be Arc'd anyway via into_user.
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum InputReadyingError {
|
||||
Parsing(Arc<SymphoniaError>),
|
||||
Creation(Arc<AudioStreamError>),
|
||||
Seeking(Arc<SymphoniaError>),
|
||||
Dropped,
|
||||
Waiting,
|
||||
NeedsSeek(SeekRequest),
|
||||
}
|
||||
|
||||
impl InputReadyingError {
|
||||
pub fn as_user(&self) -> Option<PlayError> {
|
||||
match self {
|
||||
Self::Parsing(e) => Some(PlayError::Parse(e.clone())),
|
||||
Self::Creation(e) => Some(PlayError::Create(e.clone())),
|
||||
Self::Seeking(e) => Some(PlayError::Seek(e.clone())),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn into_seek_request(self) -> Option<SeekRequest> {
|
||||
if let Self::NeedsSeek(a) = self {
|
||||
Some(a)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
105
app/vendor/songbird/src/driver/tasks/mixer/state.rs
vendored
105
app/vendor/songbird/src/driver/tasks/mixer/state.rs
vendored
@@ -1,105 +0,0 @@
|
||||
use crate::{
|
||||
constants::OPUS_PASSTHROUGH_STRIKE_LIMIT,
|
||||
driver::tasks::message::*,
|
||||
input::{Compose, Input, LiveInput, Metadata, Parsed},
|
||||
tracks::{ReadyState, SeekRequest},
|
||||
};
|
||||
use flume::Receiver;
|
||||
use rubato::FftFixedOut;
|
||||
use std::time::Instant;
|
||||
|
||||
pub enum InputState {
|
||||
NotReady(Input),
|
||||
Preparing(PreparingInfo),
|
||||
Ready(Parsed, Option<Box<dyn Compose>>),
|
||||
}
|
||||
|
||||
impl InputState {
|
||||
pub fn metadata(&mut self) -> Option<Metadata<'_>> {
|
||||
if let Self::Ready(parsed, _) = self {
|
||||
Some(parsed.into())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn ready_state(&self) -> ReadyState {
|
||||
match self {
|
||||
Self::NotReady(_) => ReadyState::Uninitialised,
|
||||
Self::Preparing(_) => ReadyState::Preparing,
|
||||
Self::Ready(_, _) => ReadyState::Playable,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Input> for InputState {
|
||||
fn from(val: Input) -> Self {
|
||||
match val {
|
||||
a @ Input::Lazy(_) => Self::NotReady(a),
|
||||
Input::Live(live, rec) => match live {
|
||||
LiveInput::Parsed(p) => Self::Ready(p, rec),
|
||||
other => Self::NotReady(Input::Live(other, rec)),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct PreparingInfo {
|
||||
#[allow(dead_code)]
|
||||
/// Time this request was fired.
|
||||
pub time: Instant,
|
||||
/// Used to handle seek requests fired while a track was being created (or a seek was in progress).
|
||||
pub queued_seek: Option<SeekRequest>,
|
||||
/// Callback from the thread pool to indicate the result of creating/parsing this track.
|
||||
pub callback: Receiver<MixerInputResultMessage>,
|
||||
}
|
||||
|
||||
pub struct DecodeState {
|
||||
pub inner_pos: usize,
|
||||
pub resampler: Option<(usize, FftFixedOut<f32>, Vec<Vec<f32>>)>,
|
||||
pub passthrough: Passthrough,
|
||||
pub passthrough_violations: u8,
|
||||
}
|
||||
|
||||
impl DecodeState {
|
||||
pub fn reset(&mut self) {
|
||||
self.inner_pos = 0;
|
||||
self.resampler = None;
|
||||
}
|
||||
|
||||
pub fn record_and_check_passthrough_strike_final(&mut self, fatal: bool) -> bool {
|
||||
self.passthrough_violations = self.passthrough_violations.saturating_add(1);
|
||||
let blocked = fatal || self.passthrough_violations > OPUS_PASSTHROUGH_STRIKE_LIMIT;
|
||||
if blocked {
|
||||
self.passthrough = Passthrough::Block;
|
||||
}
|
||||
blocked
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DecodeState {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
inner_pos: 0,
|
||||
resampler: None,
|
||||
passthrough: Passthrough::Inactive,
|
||||
passthrough_violations: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Simple state to manage decoder resets etc.
|
||||
///
|
||||
/// Inactive->Active transitions should trigger a reset.
|
||||
///
|
||||
/// Block should be used if a source contains known-bad packets:
|
||||
/// it's unlikely that packet sizes will vary, but if they do then
|
||||
/// we can't passthrough (and every attempt will trigger a codec reset,
|
||||
/// which probably won't sound too smooth).
|
||||
#[derive(Clone, Copy, Eq, PartialEq)]
|
||||
pub enum Passthrough {
|
||||
Active,
|
||||
Inactive,
|
||||
Block,
|
||||
}
|
||||
403
app/vendor/songbird/src/driver/tasks/mixer/track.rs
vendored
403
app/vendor/songbird/src/driver/tasks/mixer/track.rs
vendored
@@ -1,403 +0,0 @@
|
||||
use crate::tracks::{ReadyState, SeekRequest};
|
||||
use std::result::Result as StdResult;
|
||||
use symphonia_core::errors::Error as SymphError;
|
||||
|
||||
use super::*;
|
||||
|
||||
pub struct InternalTrack {
|
||||
pub(crate) playing: PlayMode,
|
||||
pub(crate) volume: f32,
|
||||
pub(crate) input: InputState,
|
||||
pub(crate) mix_state: DecodeState,
|
||||
pub(crate) position: Duration,
|
||||
pub(crate) play_time: Duration,
|
||||
pub(crate) commands: Receiver<TrackCommand>,
|
||||
pub(crate) loops: LoopState,
|
||||
pub(crate) callbacks: Callbacks,
|
||||
}
|
||||
|
||||
impl<'a> InternalTrack {
|
||||
pub(crate) fn decompose_track(
|
||||
val: TrackContext,
|
||||
) -> (Self, EventStore, TrackState, TrackHandle) {
|
||||
let TrackContext {
|
||||
handle,
|
||||
track,
|
||||
receiver,
|
||||
} = val;
|
||||
let out = InternalTrack {
|
||||
playing: track.playing,
|
||||
volume: track.volume,
|
||||
input: InputState::from(track.input),
|
||||
mix_state: DecodeState::default(),
|
||||
position: Duration::default(),
|
||||
play_time: Duration::default(),
|
||||
commands: receiver,
|
||||
loops: track.loops,
|
||||
callbacks: Callbacks::default(),
|
||||
};
|
||||
|
||||
let state = out.state();
|
||||
|
||||
(out, track.events, state, handle)
|
||||
}
|
||||
|
||||
pub(crate) fn state(&self) -> TrackState {
|
||||
let ready = self.input.ready_state();
|
||||
|
||||
TrackState {
|
||||
playing: self.playing.clone(),
|
||||
volume: self.volume,
|
||||
position: self.position,
|
||||
play_time: self.play_time,
|
||||
loops: self.loops,
|
||||
ready,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn view(&'a mut self) -> View<'a> {
|
||||
let ready = self.input.ready_state();
|
||||
|
||||
View {
|
||||
position: &self.position,
|
||||
play_time: &self.play_time,
|
||||
volume: &mut self.volume,
|
||||
meta: self.input.metadata(),
|
||||
ready,
|
||||
playing: &mut self.playing,
|
||||
loops: &mut self.loops,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn process_commands(&mut self, index: usize, ic: &Interconnect) -> Action {
|
||||
// Note: disconnection and an empty channel are both valid,
|
||||
// and should allow the audio object to keep running as intended.
|
||||
|
||||
// We also need to export a target seek point to the mixer, if known.
|
||||
let mut action = Action::default();
|
||||
|
||||
// Note that interconnect failures are not currently errors.
|
||||
// In correct operation, the event thread should never panic,
|
||||
// but it receiving status updates is secondary do actually
|
||||
// doing the work.
|
||||
while let Ok(cmd) = self.commands.try_recv() {
|
||||
match cmd {
|
||||
TrackCommand::Play => {
|
||||
self.playing.change_to(PlayMode::Play);
|
||||
drop(ic.events.send(EventMessage::ChangeState(
|
||||
index,
|
||||
TrackStateChange::Mode(self.playing.clone()),
|
||||
)));
|
||||
},
|
||||
TrackCommand::Pause => {
|
||||
self.playing.change_to(PlayMode::Pause);
|
||||
drop(ic.events.send(EventMessage::ChangeState(
|
||||
index,
|
||||
TrackStateChange::Mode(self.playing.clone()),
|
||||
)));
|
||||
},
|
||||
TrackCommand::Stop => {
|
||||
self.playing.change_to(PlayMode::Stop);
|
||||
drop(ic.events.send(EventMessage::ChangeState(
|
||||
index,
|
||||
TrackStateChange::Mode(self.playing.clone()),
|
||||
)));
|
||||
},
|
||||
TrackCommand::Volume(vol) => {
|
||||
self.volume = vol;
|
||||
drop(ic.events.send(EventMessage::ChangeState(
|
||||
index,
|
||||
TrackStateChange::Volume(self.volume),
|
||||
)));
|
||||
},
|
||||
TrackCommand::Seek(req) => action.seek_point = Some(req),
|
||||
TrackCommand::AddEvent(evt) => {
|
||||
drop(ic.events.send(EventMessage::AddTrackEvent(index, evt)));
|
||||
},
|
||||
TrackCommand::Do(func) => {
|
||||
if let Some(indiv_action) = func(self.view()) {
|
||||
action.combine(indiv_action);
|
||||
}
|
||||
|
||||
drop(ic.events.send(EventMessage::ChangeState(
|
||||
index,
|
||||
TrackStateChange::Total(self.state()),
|
||||
)));
|
||||
},
|
||||
TrackCommand::Request(tx) => {
|
||||
drop(tx.send(self.state()));
|
||||
},
|
||||
TrackCommand::Loop(loops) => {
|
||||
self.loops = loops;
|
||||
drop(ic.events.send(EventMessage::ChangeState(
|
||||
index,
|
||||
TrackStateChange::Loops(self.loops, true),
|
||||
)));
|
||||
},
|
||||
TrackCommand::MakePlayable(callback) => action.make_playable = Some(callback),
|
||||
}
|
||||
}
|
||||
|
||||
action
|
||||
}
|
||||
|
||||
pub(crate) fn do_loop(&mut self) -> bool {
|
||||
match &mut self.loops {
|
||||
LoopState::Infinite => true,
|
||||
LoopState::Finite(n) =>
|
||||
if let Some(new_n) = nonmax::NonMaxU32::new(n.get().wrapping_sub(1)) {
|
||||
*n = new_n;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Steps playback location forward by one frame.
|
||||
pub(crate) fn step_frame(&mut self) {
|
||||
self.position += TIMESTEP_LENGTH;
|
||||
self.play_time += TIMESTEP_LENGTH;
|
||||
}
|
||||
|
||||
pub(crate) fn should_check_input(&self) -> bool {
|
||||
self.playing.is_playing() || matches!(self.input, InputState::Preparing(_))
|
||||
}
|
||||
|
||||
pub(crate) fn end(&mut self) -> &mut Self {
|
||||
self.playing.change_to(PlayMode::End);
|
||||
|
||||
self
|
||||
}
|
||||
|
||||
/// Readies the requested input state.
|
||||
///
|
||||
/// Returns the usable version of the audio if available, and whether the track should be deleted.
|
||||
pub(crate) fn get_or_ready_input(
|
||||
&'a mut self,
|
||||
id: usize,
|
||||
interconnect: &Interconnect,
|
||||
pool: &BlockyTaskPool,
|
||||
config: &Arc<Config>,
|
||||
prevent_events: bool,
|
||||
) -> StdResult<(&'a mut Parsed, &'a mut DecodeState), InputReadyingError> {
|
||||
let input = &mut self.input;
|
||||
let mix_state = &mut self.mix_state;
|
||||
|
||||
let (out, queued_seek) = match input {
|
||||
InputState::NotReady(_) => {
|
||||
let (tx, rx) = flume::bounded(1);
|
||||
|
||||
let mut state = InputState::Preparing(PreparingInfo {
|
||||
time: Instant::now(),
|
||||
queued_seek: None,
|
||||
callback: rx,
|
||||
});
|
||||
|
||||
std::mem::swap(&mut state, input);
|
||||
|
||||
match state {
|
||||
InputState::NotReady(a @ Input::Lazy(_)) => {
|
||||
pool.create(tx, a, None, config.clone());
|
||||
},
|
||||
InputState::NotReady(Input::Live(audio, rec)) => {
|
||||
pool.parse(config.clone(), tx, audio, rec, None);
|
||||
},
|
||||
_ => unreachable!(),
|
||||
}
|
||||
|
||||
if !prevent_events {
|
||||
drop(interconnect.events.send(EventMessage::ChangeState(
|
||||
id,
|
||||
TrackStateChange::Ready(ReadyState::Preparing),
|
||||
)));
|
||||
}
|
||||
|
||||
(Err(InputReadyingError::Waiting), None)
|
||||
},
|
||||
InputState::Preparing(info) => {
|
||||
let queued_seek = info.queued_seek.take();
|
||||
|
||||
let orig_out = match info.callback.try_recv() {
|
||||
Ok(MixerInputResultMessage::Built(parsed, rec)) => {
|
||||
*input = InputState::Ready(parsed, rec);
|
||||
mix_state.reset();
|
||||
|
||||
// possible TODO: set position to the true track position here?
|
||||
// ISSUE: need to get next_packet to see its `ts`, but inner_pos==0
|
||||
// will trigger next packet to be taken at mix time.
|
||||
|
||||
if !prevent_events {
|
||||
drop(interconnect.events.send(EventMessage::ChangeState(
|
||||
id,
|
||||
TrackStateChange::Ready(ReadyState::Playable),
|
||||
)));
|
||||
}
|
||||
|
||||
self.callbacks.playable();
|
||||
|
||||
if let InputState::Ready(ref mut parsed, _) = input {
|
||||
Ok(parsed)
|
||||
} else {
|
||||
unreachable!()
|
||||
}
|
||||
},
|
||||
Ok(MixerInputResultMessage::Seek(parsed, rec, seek_res)) => {
|
||||
match seek_res {
|
||||
Ok(pos) => {
|
||||
if let Some(time_base) = parsed.decoder.codec_params().time_base {
|
||||
// Update track's position to match the actual timestamp the
|
||||
// seek landed at.
|
||||
let new_time = time_base.calc_time(pos.actual_ts);
|
||||
let time_in_float = new_time.seconds as f64 + new_time.frac;
|
||||
self.position =
|
||||
std::time::Duration::from_secs_f64(time_in_float);
|
||||
|
||||
self.callbacks.seeked(self.position);
|
||||
self.callbacks.playable();
|
||||
|
||||
if !prevent_events {
|
||||
drop(interconnect.events.send(EventMessage::ChangeState(
|
||||
id,
|
||||
TrackStateChange::Position(self.position),
|
||||
)));
|
||||
|
||||
drop(interconnect.events.send(EventMessage::ChangeState(
|
||||
id,
|
||||
TrackStateChange::Ready(ReadyState::Playable),
|
||||
)));
|
||||
}
|
||||
|
||||
// Our decoder state etc. must be reset.
|
||||
// (Symphonia decoder state reset in the thread pool during
|
||||
// the operation.)
|
||||
mix_state.reset();
|
||||
*input = InputState::Ready(parsed, rec);
|
||||
|
||||
if let InputState::Ready(ref mut parsed, _) = input {
|
||||
Ok(parsed)
|
||||
} else {
|
||||
unreachable!()
|
||||
}
|
||||
} else {
|
||||
Err(InputReadyingError::Seeking(
|
||||
SymphError::Unsupported("Track had no recorded time base.")
|
||||
.into(),
|
||||
))
|
||||
}
|
||||
},
|
||||
Err(e) => Err(InputReadyingError::Seeking(e)),
|
||||
}
|
||||
},
|
||||
Ok(MixerInputResultMessage::CreateErr(e)) =>
|
||||
Err(InputReadyingError::Creation(e)),
|
||||
Ok(MixerInputResultMessage::ParseErr(e)) => Err(InputReadyingError::Parsing(e)),
|
||||
Err(TryRecvError::Disconnected) => Err(InputReadyingError::Dropped),
|
||||
Err(TryRecvError::Empty) => Err(InputReadyingError::Waiting),
|
||||
};
|
||||
|
||||
let orig_out = orig_out.map(|a| (a, mix_state));
|
||||
|
||||
if let Err(ref e) = orig_out {
|
||||
if let Some(e) = e.as_user() {
|
||||
self.callbacks.readying_error(e);
|
||||
}
|
||||
}
|
||||
|
||||
(orig_out, queued_seek)
|
||||
},
|
||||
InputState::Ready(ref mut parsed, _) => (Ok((parsed, mix_state)), None),
|
||||
};
|
||||
|
||||
match (out, queued_seek) {
|
||||
(Ok(_), Some(request)) => Err(InputReadyingError::NeedsSeek(request)),
|
||||
(a, _) => a,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn seek(
|
||||
&mut self,
|
||||
id: usize,
|
||||
request: SeekRequest,
|
||||
interconnect: &Interconnect,
|
||||
pool: &BlockyTaskPool,
|
||||
config: &Arc<Config>,
|
||||
prevent_events: bool,
|
||||
) {
|
||||
if let InputState::Preparing(p) = &mut self.input {
|
||||
p.queued_seek = Some(request);
|
||||
return;
|
||||
}
|
||||
|
||||
// might be a little topsy turvy: rethink me.
|
||||
let SeekRequest { time, callback } = request;
|
||||
|
||||
self.callbacks.seek = Some(callback);
|
||||
if !prevent_events {
|
||||
drop(interconnect.events.send(EventMessage::ChangeState(
|
||||
id,
|
||||
TrackStateChange::Ready(ReadyState::Preparing),
|
||||
)));
|
||||
}
|
||||
|
||||
let backseek_needed = time < self.position;
|
||||
|
||||
let time = Time::from(time.as_secs_f64());
|
||||
let mut ts = SeekTo::Time {
|
||||
time,
|
||||
track_id: None,
|
||||
};
|
||||
let (tx, rx) = flume::bounded(1);
|
||||
|
||||
let state = std::mem::replace(
|
||||
&mut self.input,
|
||||
InputState::Preparing(PreparingInfo {
|
||||
time: Instant::now(),
|
||||
callback: rx,
|
||||
queued_seek: None,
|
||||
}),
|
||||
);
|
||||
|
||||
match state {
|
||||
InputState::Ready(p, r) => {
|
||||
if let SeekTo::Time { time: _, track_id } = &mut ts {
|
||||
*track_id = Some(p.track_id);
|
||||
}
|
||||
|
||||
pool.seek(tx, p, r, ts, backseek_needed, config.clone());
|
||||
},
|
||||
InputState::NotReady(lazy) => pool.create(tx, lazy, Some(ts), config.clone()),
|
||||
InputState::Preparing(_) => unreachable!(), // Covered above.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Callbacks {
|
||||
pub seek: Option<Sender<StdResult<Duration, PlayError>>>,
|
||||
pub make_playable: Option<Sender<StdResult<(), PlayError>>>,
|
||||
}
|
||||
|
||||
impl Callbacks {
|
||||
fn readying_error(&mut self, err: PlayError) {
|
||||
if let Some(callback) = self.seek.take() {
|
||||
drop(callback.send(Err(err.clone())));
|
||||
}
|
||||
|
||||
if let Some(callback) = self.make_playable.take() {
|
||||
drop(callback.send(Err(err)));
|
||||
}
|
||||
}
|
||||
|
||||
fn playable(&mut self) {
|
||||
if let Some(callback) = self.make_playable.take() {
|
||||
drop(callback.send(Ok(())));
|
||||
}
|
||||
}
|
||||
|
||||
fn seeked(&mut self, time: Duration) {
|
||||
if let Some(callback) = self.seek.take() {
|
||||
drop(callback.send(Ok(time)));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
use symphonia_core::{formats::SeekTo, units::Time};
|
||||
|
||||
// SeekTo lacks Copy and Clone... somehow.
|
||||
pub fn copy_seek_to(pos: &SeekTo) -> SeekTo {
|
||||
match *pos {
|
||||
SeekTo::Time { time, track_id } => SeekTo::Time { time, track_id },
|
||||
SeekTo::TimeStamp { ts, track_id } => SeekTo::TimeStamp { ts, track_id },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn seek_to_is_zero(pos: &SeekTo) -> bool {
|
||||
match *pos {
|
||||
SeekTo::Time { time, .. } =>
|
||||
time == Time {
|
||||
seconds: 0,
|
||||
frac: 0.0,
|
||||
},
|
||||
SeekTo::TimeStamp { ts, .. } => ts == 0,
|
||||
}
|
||||
}
|
||||
351
app/vendor/songbird/src/driver/tasks/mod.rs
vendored
351
app/vendor/songbird/src/driver/tasks/mod.rs
vendored
@@ -1,351 +0,0 @@
|
||||
#![allow(missing_docs)]
|
||||
|
||||
pub(crate) mod disposal;
|
||||
pub mod error;
|
||||
mod events;
|
||||
pub mod message;
|
||||
pub mod mixer;
|
||||
#[cfg(feature = "receive")]
|
||||
pub(crate) mod udp_rx;
|
||||
pub(crate) mod ws;
|
||||
|
||||
use super::connection::{error::Error as ConnectionError, Connection};
|
||||
use crate::{
|
||||
events::{
|
||||
context_data::{DisconnectKind, DisconnectReason},
|
||||
internal_data::{InternalConnect, InternalDisconnect},
|
||||
CoreContext,
|
||||
},
|
||||
Config,
|
||||
ConnectionInfo,
|
||||
FloatDuration,
|
||||
};
|
||||
use flume::{Receiver, Sender};
|
||||
use message::*;
|
||||
use tokio::{spawn, time::sleep as tsleep};
|
||||
use tracing::{debug, instrument, trace};
|
||||
|
||||
pub(crate) fn start(config: Config, rx: Receiver<CoreMessage>, tx: Sender<CoreMessage>) {
|
||||
spawn(async move {
|
||||
trace!("Driver started.");
|
||||
runner(config, rx, tx).await;
|
||||
trace!("Driver finished.");
|
||||
});
|
||||
}
|
||||
|
||||
fn start_internals(core: Sender<CoreMessage>, config: &Config) -> Interconnect {
|
||||
let (evt_tx, evt_rx) = flume::unbounded();
|
||||
let (mix_tx, mix_rx) = flume::unbounded();
|
||||
|
||||
spawn(async move {
|
||||
trace!("Event processor started.");
|
||||
events::runner(evt_rx).await;
|
||||
trace!("Event processor finished.");
|
||||
});
|
||||
|
||||
let ic = Interconnect {
|
||||
core,
|
||||
events: evt_tx,
|
||||
mixer: mix_tx,
|
||||
};
|
||||
|
||||
config.get_scheduler().new_mixer(config, ic.clone(), mix_rx);
|
||||
ic
|
||||
}
|
||||
|
||||
#[instrument(skip(rx, tx))]
|
||||
async fn runner(mut config: Config, rx: Receiver<CoreMessage>, tx: Sender<CoreMessage>) {
|
||||
let mut next_config: Option<Config> = None;
|
||||
let mut connection: Option<Connection> = None;
|
||||
let mut interconnect = start_internals(tx, &config);
|
||||
let mut retrying = None;
|
||||
let mut attempt_idx = 0;
|
||||
|
||||
while let Ok(msg) = rx.recv_async().await {
|
||||
match msg {
|
||||
CoreMessage::ConnectWithResult(info, tx) => {
|
||||
config = if let Some(new_config) = next_config.take() {
|
||||
drop(
|
||||
interconnect
|
||||
.mixer
|
||||
.send(MixerMessage::SetConfig(new_config.clone())),
|
||||
);
|
||||
new_config
|
||||
} else {
|
||||
config
|
||||
};
|
||||
|
||||
if connection.as_ref().is_none_or(|conn| conn.info != info) {
|
||||
// Only *actually* reconnect if the conn info changed, or we don't have an
|
||||
// active connection.
|
||||
// This allows the gateway component to keep sending join requests independent
|
||||
// of driver failures.
|
||||
connection = ConnectionRetryData::connect(tx, info, &mut attempt_idx)
|
||||
.attempt(&mut retrying, &interconnect, &config)
|
||||
.await;
|
||||
} else {
|
||||
// No reconnection was attempted as there's a valid, identical connection;
|
||||
// tell the outside listener that the operation was a success.
|
||||
drop(tx.send(Ok(())));
|
||||
}
|
||||
},
|
||||
CoreMessage::RetryConnect(retry_idx) => {
|
||||
debug!("Retrying idx: {} (vs. {})", retry_idx, attempt_idx);
|
||||
if retry_idx == attempt_idx {
|
||||
if let Some(progress) = retrying.take() {
|
||||
connection = progress
|
||||
.attempt(&mut retrying, &interconnect, &config)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
},
|
||||
CoreMessage::Disconnect => {
|
||||
let last_conn = connection.take();
|
||||
drop(interconnect.mixer.send(MixerMessage::DropConn));
|
||||
drop(interconnect.mixer.send(MixerMessage::RebuildEncoder));
|
||||
|
||||
if let Some(conn) = last_conn {
|
||||
drop(interconnect.events.send(EventMessage::FireCoreEvent(
|
||||
CoreContext::DriverDisconnect(InternalDisconnect {
|
||||
kind: DisconnectKind::Runtime,
|
||||
reason: Some(DisconnectReason::Requested),
|
||||
info: conn.info.clone(),
|
||||
}),
|
||||
)));
|
||||
}
|
||||
},
|
||||
CoreMessage::SignalWsClosure(ws_idx, ws_info, mut reason) => {
|
||||
// if idx is not a match, quash reason
|
||||
// (i.e., prevent users from mistakenly trying to reconnect for an *old* dead conn).
|
||||
// if it *is* a match, the conn needs to die!
|
||||
// (as the WS channel has truly given up the ghost).
|
||||
let conn = if ws_idx == attempt_idx {
|
||||
drop(interconnect.mixer.send(MixerMessage::DropConn));
|
||||
drop(interconnect.mixer.send(MixerMessage::RebuildEncoder));
|
||||
connection.take()
|
||||
} else {
|
||||
reason = None;
|
||||
None
|
||||
};
|
||||
|
||||
// Conn may have been unset earlier (i.e., in a deliberate disconnect).
|
||||
// If so, do not repropagate/repeat the disconnect event.
|
||||
if conn.is_some() {
|
||||
drop(interconnect.events.send(EventMessage::FireCoreEvent(
|
||||
CoreContext::DriverDisconnect(InternalDisconnect {
|
||||
kind: DisconnectKind::Runtime,
|
||||
reason,
|
||||
info: ws_info,
|
||||
}),
|
||||
)));
|
||||
}
|
||||
},
|
||||
CoreMessage::SetTrack(s) => {
|
||||
drop(interconnect.mixer.send(MixerMessage::SetTrack(s)));
|
||||
},
|
||||
CoreMessage::AddTrack(s) => {
|
||||
drop(interconnect.mixer.send(MixerMessage::AddTrack(s)));
|
||||
},
|
||||
CoreMessage::SetBitrate(b) => {
|
||||
drop(interconnect.mixer.send(MixerMessage::SetBitrate(b)));
|
||||
},
|
||||
CoreMessage::SetConfig(mut new_config) => {
|
||||
next_config = Some(new_config.clone());
|
||||
|
||||
new_config.make_safe(&config, connection.is_some());
|
||||
|
||||
drop(interconnect.mixer.send(MixerMessage::SetConfig(new_config)));
|
||||
},
|
||||
CoreMessage::AddEvent(evt) => {
|
||||
drop(interconnect.events.send(EventMessage::AddGlobalEvent(evt)));
|
||||
},
|
||||
CoreMessage::RemoveGlobalEvents => {
|
||||
drop(interconnect.events.send(EventMessage::RemoveGlobalEvents));
|
||||
},
|
||||
CoreMessage::Mute(m) => {
|
||||
drop(interconnect.mixer.send(MixerMessage::SetMute(m)));
|
||||
},
|
||||
CoreMessage::Reconnect => {
|
||||
if let Some(mut conn) = connection.take() {
|
||||
// try once: if interconnect, try again.
|
||||
// if still issue, full connect.
|
||||
let info = conn.info.clone();
|
||||
|
||||
let full_connect = match conn.reconnect(&config).await {
|
||||
Ok(()) => {
|
||||
connection = Some(conn);
|
||||
false
|
||||
},
|
||||
Err(ConnectionError::InterconnectFailure(_)) => {
|
||||
interconnect.restart_volatile_internals();
|
||||
|
||||
match conn.reconnect(&config).await {
|
||||
Ok(()) => {
|
||||
connection = Some(conn);
|
||||
false
|
||||
},
|
||||
_ => true,
|
||||
}
|
||||
},
|
||||
_ => true,
|
||||
};
|
||||
|
||||
if full_connect {
|
||||
connection = ConnectionRetryData::reconnect(info, &mut attempt_idx)
|
||||
.attempt(&mut retrying, &interconnect, &config)
|
||||
.await;
|
||||
} else if let Some(ref connection) = &connection {
|
||||
drop(interconnect.events.send(EventMessage::FireCoreEvent(
|
||||
CoreContext::DriverReconnect(InternalConnect {
|
||||
info: connection.info.clone(),
|
||||
ssrc: connection.ssrc,
|
||||
}),
|
||||
)));
|
||||
}
|
||||
}
|
||||
},
|
||||
CoreMessage::FullReconnect =>
|
||||
if let Some(conn) = connection.take() {
|
||||
let info = conn.info.clone();
|
||||
|
||||
connection = ConnectionRetryData::reconnect(info, &mut attempt_idx)
|
||||
.attempt(&mut retrying, &interconnect, &config)
|
||||
.await;
|
||||
},
|
||||
CoreMessage::RebuildInterconnect => {
|
||||
interconnect.restart_volatile_internals();
|
||||
},
|
||||
CoreMessage::Poison => break,
|
||||
}
|
||||
}
|
||||
|
||||
trace!("Main thread exited");
|
||||
interconnect.poison_all();
|
||||
}
|
||||
|
||||
struct ConnectionRetryData {
|
||||
flavour: ConnectionFlavour,
|
||||
attempts: u8,
|
||||
last_wait: Option<FloatDuration>,
|
||||
info: ConnectionInfo,
|
||||
idx: usize,
|
||||
}
|
||||
|
||||
impl ConnectionRetryData {
|
||||
fn connect(
|
||||
tx: Sender<Result<(), ConnectionError>>,
|
||||
info: ConnectionInfo,
|
||||
idx_src: &mut usize,
|
||||
) -> Self {
|
||||
Self::base(ConnectionFlavour::Connect(tx), info, idx_src)
|
||||
}
|
||||
|
||||
fn reconnect(info: ConnectionInfo, idx_src: &mut usize) -> Self {
|
||||
Self::base(ConnectionFlavour::Reconnect, info, idx_src)
|
||||
}
|
||||
|
||||
fn base(flavour: ConnectionFlavour, info: ConnectionInfo, idx_src: &mut usize) -> Self {
|
||||
*idx_src = idx_src.wrapping_add(1);
|
||||
|
||||
Self {
|
||||
flavour,
|
||||
attempts: 0,
|
||||
last_wait: None,
|
||||
info,
|
||||
idx: *idx_src,
|
||||
}
|
||||
}
|
||||
|
||||
async fn attempt(
|
||||
mut self,
|
||||
attempt_slot: &mut Option<Self>,
|
||||
interconnect: &Interconnect,
|
||||
config: &Config,
|
||||
) -> Option<Connection> {
|
||||
match Connection::new(self.info.clone(), interconnect, config, self.idx).await {
|
||||
Ok(connection) => {
|
||||
match self.flavour {
|
||||
ConnectionFlavour::Connect(tx) => {
|
||||
// Other side may not be listening: this is fine.
|
||||
drop(tx.send(Ok(())));
|
||||
|
||||
drop(interconnect.events.send(EventMessage::FireCoreEvent(
|
||||
CoreContext::DriverConnect(InternalConnect {
|
||||
info: connection.info.clone(),
|
||||
ssrc: connection.ssrc,
|
||||
}),
|
||||
)));
|
||||
},
|
||||
ConnectionFlavour::Reconnect => {
|
||||
drop(interconnect.events.send(EventMessage::FireCoreEvent(
|
||||
CoreContext::DriverReconnect(InternalConnect {
|
||||
info: connection.info.clone(),
|
||||
ssrc: connection.ssrc,
|
||||
}),
|
||||
)));
|
||||
},
|
||||
}
|
||||
|
||||
Some(connection)
|
||||
},
|
||||
Err(why) => {
|
||||
debug!("Failed to connect for {:?}: {}", self.info.guild_id, why);
|
||||
if let Some(t) = config.driver_retry.retry_in(self.last_wait, self.attempts) {
|
||||
let remote_ic = interconnect.clone();
|
||||
let idx = self.idx;
|
||||
|
||||
spawn(async move {
|
||||
tsleep(t.into()).await;
|
||||
drop(remote_ic.core.send(CoreMessage::RetryConnect(idx)));
|
||||
});
|
||||
|
||||
self.attempts += 1;
|
||||
self.last_wait = Some(t);
|
||||
|
||||
debug!(
|
||||
"Retrying connection for {:?} in {}s ({}/{:?})",
|
||||
self.info.guild_id,
|
||||
t.as_secs_f32(),
|
||||
self.attempts,
|
||||
config.driver_retry.retry_limit
|
||||
);
|
||||
|
||||
*attempt_slot = Some(self);
|
||||
} else {
|
||||
let reason = Some(DisconnectReason::from(&why));
|
||||
|
||||
match self.flavour {
|
||||
ConnectionFlavour::Connect(tx) => {
|
||||
// See above.
|
||||
drop(tx.send(Err(why)));
|
||||
|
||||
drop(interconnect.events.send(EventMessage::FireCoreEvent(
|
||||
CoreContext::DriverDisconnect(InternalDisconnect {
|
||||
kind: DisconnectKind::Connect,
|
||||
reason,
|
||||
info: self.info,
|
||||
}),
|
||||
)));
|
||||
},
|
||||
ConnectionFlavour::Reconnect => {
|
||||
drop(interconnect.events.send(EventMessage::FireCoreEvent(
|
||||
CoreContext::DriverDisconnect(InternalDisconnect {
|
||||
kind: DisconnectKind::Reconnect,
|
||||
reason,
|
||||
info: self.info,
|
||||
}),
|
||||
)));
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum ConnectionFlavour {
|
||||
Connect(Sender<Result<(), ConnectionError>>),
|
||||
Reconnect,
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
use crate::constants::STEREO_FRAME_SIZE;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum PacketDecodeSize {
|
||||
/// Minimum frame size on Discord.
|
||||
TwentyMillis,
|
||||
/// Hybrid packet, sent by Firefox web client.
|
||||
///
|
||||
/// Likely 20ms frame + 10ms frame.
|
||||
ThirtyMillis,
|
||||
/// Next largest frame size.
|
||||
FortyMillis,
|
||||
/// Maximum Opus frame size.
|
||||
SixtyMillis,
|
||||
/// Maximum Opus packet size: 120ms.
|
||||
Max,
|
||||
}
|
||||
|
||||
impl PacketDecodeSize {
|
||||
pub fn bump_up(self) -> Self {
|
||||
match self {
|
||||
Self::TwentyMillis => Self::ThirtyMillis,
|
||||
Self::ThirtyMillis => Self::FortyMillis,
|
||||
Self::FortyMillis => Self::SixtyMillis,
|
||||
Self::SixtyMillis | Self::Max => Self::Max,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn can_bump_up(self) -> bool {
|
||||
self != Self::Max
|
||||
}
|
||||
|
||||
pub fn len(self) -> usize {
|
||||
match self {
|
||||
Self::TwentyMillis => STEREO_FRAME_SIZE,
|
||||
Self::ThirtyMillis => (STEREO_FRAME_SIZE / 2) * 3,
|
||||
Self::FortyMillis => 2 * STEREO_FRAME_SIZE,
|
||||
Self::SixtyMillis => 3 * STEREO_FRAME_SIZE,
|
||||
Self::Max => 6 * STEREO_FRAME_SIZE,
|
||||
}
|
||||
}
|
||||
}
|
||||
120
app/vendor/songbird/src/driver/tasks/udp_rx/held.rs
vendored
120
app/vendor/songbird/src/driver/tasks/udp_rx/held.rs
vendored
@@ -1,120 +0,0 @@
|
||||
//! Jarvis patch: RTP packets whose SSRC Discord has not named yet are held for a moment.
|
||||
//!
|
||||
//! Discord names a speaker's SSRC in a Speaking op over the voice websocket the first time they speak,
|
||||
//! and that op arrives after their first packets over UDP. Unpatched, songbird dropped DAVE packets
|
||||
//! of an unnamed SSRC (it needs the user to decrypt them), which cut the first ~100 ms off the first
|
||||
//! thing each person said after a join ("자비스" heard as "아비"). Held packets are processed in
|
||||
//! arrival order before the first packet after the name arrives, or on the next 20 ms playout tick
|
||||
//! when no packet follows (a short utterance that ended before its name came).
|
||||
|
||||
use bytes::BytesMut;
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::time::Duration;
|
||||
use tokio::time::Instant;
|
||||
|
||||
/// A packet older than this when its SSRC is named is dropped.
|
||||
pub(crate) const HOLD: Duration = Duration::from_secs(1);
|
||||
/// At most this many packets per SSRC: one second of 20 ms packets.
|
||||
pub(crate) const MAX_PACKETS: usize = 50;
|
||||
/// At most this many unnamed SSRCs at once.
|
||||
pub(crate) const MAX_SSRCS: usize = 32;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct Held {
|
||||
by_ssrc: HashMap<u32, VecDeque<(Instant, BytesMut)>>,
|
||||
}
|
||||
|
||||
impl Held {
|
||||
pub(crate) fn hold(&mut self, ssrc: u32, at: Instant, packet: BytesMut) {
|
||||
if !self.by_ssrc.contains_key(&ssrc) && self.by_ssrc.len() >= MAX_SSRCS {
|
||||
return;
|
||||
}
|
||||
let queue = self.by_ssrc.entry(ssrc).or_default();
|
||||
while queue.front().is_some_and(|(t, _)| at.duration_since(*t) > HOLD) || queue.len() >= MAX_PACKETS {
|
||||
queue.pop_front();
|
||||
}
|
||||
queue.push_back((at, packet));
|
||||
}
|
||||
|
||||
/// The packets held for `ssrc` that are at most `HOLD` old, oldest first. Forgets all of them.
|
||||
pub(crate) fn take(&mut self, ssrc: u32, now: Instant) -> Vec<BytesMut> {
|
||||
self.by_ssrc
|
||||
.remove(&ssrc)
|
||||
.map(|q| q.into_iter().filter(|(t, _)| now.duration_since(*t) <= HOLD).map(|(_, p)| p).collect())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// The SSRCs with held packets.
|
||||
pub(crate) fn ssrcs(&self) -> impl Iterator<Item = u32> + '_ {
|
||||
self.by_ssrc.keys().copied()
|
||||
}
|
||||
|
||||
/// Forgets every SSRC whose newest packet is older than `HOLD`.
|
||||
pub(crate) fn prune(&mut self, now: Instant) {
|
||||
self.by_ssrc.retain(|_, q| q.back().is_some_and(|(t, _)| now.duration_since(*t) <= HOLD));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn p(n: u8) -> BytesMut {
|
||||
BytesMut::from(&[n][..])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn packets_held_before_the_name_come_back_in_arrival_order() {
|
||||
let mut h = Held::default();
|
||||
let t = Instant::now();
|
||||
for n in 0..3 {
|
||||
h.hold(7, t + Duration::from_millis(20 * u64::from(n)), p(n));
|
||||
}
|
||||
h.hold(8, t, p(9));
|
||||
assert_eq!(h.take(7, t + Duration::from_millis(100)), vec![p(0), p(1), p(2)]);
|
||||
assert!(h.take(7, t + Duration::from_millis(100)).is_empty(), "taken once");
|
||||
assert_eq!(h.take(8, t), vec![p(9)], "other speakers are kept apart");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn packets_older_than_the_hold_are_dropped() {
|
||||
let mut h = Held::default();
|
||||
let t = Instant::now();
|
||||
h.hold(7, t, p(0));
|
||||
h.hold(7, t + Duration::from_millis(900), p(1));
|
||||
assert_eq!(h.take(7, t + Duration::from_millis(1500)), vec![p(1)]);
|
||||
|
||||
h.hold(7, t, p(0));
|
||||
h.hold(7, t + HOLD + Duration::from_millis(20), p(1));
|
||||
assert_eq!(h.take(7, t + HOLD + Duration::from_millis(20)), vec![p(1)], "dropped as newer ones come");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn at_most_a_second_of_packets_per_speaker_and_a_bounded_number_of_speakers() {
|
||||
let mut h = Held::default();
|
||||
let t = Instant::now();
|
||||
for n in 0..(MAX_PACKETS as u8 + 5) {
|
||||
h.hold(7, t, p(n));
|
||||
}
|
||||
let kept = h.take(7, t);
|
||||
assert_eq!(kept.len(), MAX_PACKETS);
|
||||
assert_eq!(kept[0], p(5), "the newest are kept");
|
||||
|
||||
for ssrc in 0..(MAX_SSRCS as u32 + 3) {
|
||||
h.hold(100 + ssrc, t, p(1));
|
||||
}
|
||||
assert!(h.take(100 + MAX_SSRCS as u32, t).is_empty());
|
||||
assert_eq!(h.take(100, t), vec![p(1)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prune_forgets_speakers_that_stopped_sending() {
|
||||
let mut h = Held::default();
|
||||
let t = Instant::now();
|
||||
h.hold(7, t, p(0));
|
||||
h.hold(8, t + Duration::from_millis(1500), p(1));
|
||||
h.prune(t + Duration::from_millis(1600));
|
||||
assert!(h.take(7, t).is_empty());
|
||||
assert_eq!(h.take(8, t + Duration::from_millis(1600)), vec![p(1)]);
|
||||
}
|
||||
}
|
||||
485
app/vendor/songbird/src/driver/tasks/udp_rx/mod.rs
vendored
485
app/vendor/songbird/src/driver/tasks/udp_rx/mod.rs
vendored
@@ -1,485 +0,0 @@
|
||||
mod decode_sizes;
|
||||
mod held;
|
||||
mod playout_buffer;
|
||||
mod ssrc_state;
|
||||
|
||||
use self::{decode_sizes::*, held::Held, playout_buffer::*, ssrc_state::*};
|
||||
|
||||
use super::message::*;
|
||||
use crate::driver::{CryptoMode, DecodeMode};
|
||||
use crate::{
|
||||
constants::*,
|
||||
driver::crypto::Cipher,
|
||||
events::{context_data::VoiceTick, internal_data::*, CoreContext},
|
||||
Config,
|
||||
};
|
||||
use bytes::BytesMut;
|
||||
use discortp::{
|
||||
demux::{self, DemuxedMut},
|
||||
rtp::RtpPacket,
|
||||
};
|
||||
use discortp::{MutablePacket, Packet};
|
||||
use flume::Receiver;
|
||||
use std::sync::{
|
||||
atomic::{AtomicU16, Ordering},
|
||||
RwLock,
|
||||
};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
num::Wrapping,
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{net::UdpSocket, select, time::Instant};
|
||||
use tracing::{error, instrument, trace, warn};
|
||||
|
||||
type RtpSequence = Wrapping<u16>;
|
||||
type RtpTimestamp = Wrapping<u32>;
|
||||
type RtpSsrc = u32;
|
||||
|
||||
struct UdpRx {
|
||||
cipher: Cipher,
|
||||
crypto_mode: CryptoMode,
|
||||
decoder_map: HashMap<RtpSsrc, SsrcState>,
|
||||
config: Config,
|
||||
rx: Receiver<UdpRxMessage>,
|
||||
ssrc_signalling: Arc<SsrcTracker>,
|
||||
udp_socket: UdpSocket,
|
||||
dave_session: Arc<RwLock<Option<davey::DaveSession>>>,
|
||||
dave_protocol_version: Arc<AtomicU16>,
|
||||
/// Jarvis patch: packets of SSRCs not named yet (see held.rs).
|
||||
held: Held,
|
||||
}
|
||||
|
||||
impl UdpRx {
|
||||
#[instrument(skip(self))]
|
||||
async fn run(&mut self, interconnect: &mut Interconnect) {
|
||||
let mut cleanup_time = Instant::now();
|
||||
let mut playout_time = Instant::now() + TIMESTEP_LENGTH;
|
||||
let mut byte_dest: Option<BytesMut> = None;
|
||||
|
||||
loop {
|
||||
if byte_dest.is_none() {
|
||||
byte_dest = Some(BytesMut::zeroed(VOICE_PACKET_MAX));
|
||||
}
|
||||
|
||||
select! {
|
||||
Ok((len, _addr)) = self.udp_socket.recv_from(byte_dest.as_mut().unwrap()) => {
|
||||
let mut pkt = byte_dest.take().unwrap();
|
||||
pkt.truncate(len);
|
||||
|
||||
self.process_udp_message(interconnect, pkt).await;
|
||||
},
|
||||
msg = self.rx.recv_async() => {
|
||||
match msg {
|
||||
Ok(UdpRxMessage::ReplaceInterconnect(i)) => {
|
||||
*interconnect = i;
|
||||
},
|
||||
Ok(UdpRxMessage::SetConfig(new_config)) => {
|
||||
if let DecodeMode::Decode(old_config) = &mut self.config.decode_mode {
|
||||
if *old_config != new_config {
|
||||
*old_config = new_config;
|
||||
self.decoder_map.values_mut().for_each(|v| v.reconfigure_decoder(new_config));
|
||||
}
|
||||
}
|
||||
},
|
||||
Err(flume::RecvError::Disconnected) => break,
|
||||
}
|
||||
},
|
||||
() = tokio::time::sleep_until(playout_time) => {
|
||||
// Jarvis patch: a short first utterance may have no packet after its name came.
|
||||
self.release_named(interconnect).await;
|
||||
|
||||
let mut tick = VoiceTick {
|
||||
speaking: HashMap::new(),
|
||||
silent: HashSet::new(),
|
||||
};
|
||||
|
||||
for (ssrc, state) in &mut self.decoder_map {
|
||||
match state.get_voice_tick(&self.config) {
|
||||
Ok(Some(data)) => {
|
||||
tick.speaking.insert(*ssrc, data);
|
||||
},
|
||||
Ok(None) => {
|
||||
if !state.disconnected {
|
||||
tick.silent.insert(*ssrc);
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
warn!("Decode error for SSRC {ssrc}: {e:?}");
|
||||
tick.silent.insert(*ssrc);
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
playout_time += TIMESTEP_LENGTH;
|
||||
|
||||
drop(interconnect.events.send(EventMessage::FireCoreEvent(CoreContext::VoiceTick(tick))));
|
||||
},
|
||||
() = tokio::time::sleep_until(cleanup_time) => {
|
||||
// periodic cleanup.
|
||||
let now = Instant::now();
|
||||
|
||||
// check ssrc map to see if the WS task has informed us of any disconnects.
|
||||
loop {
|
||||
// This is structured in an odd way to prevent deadlocks.
|
||||
// while-let seemed to keep the dashmap iter() alive for block scope, rather than
|
||||
// just the initialiser.
|
||||
let id = {
|
||||
if let Some(id) = self.ssrc_signalling.disconnected_users.iter().next().map(|v| *v.key()) {
|
||||
id
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
_ = self.ssrc_signalling.disconnected_users.remove(&id);
|
||||
if let Some((_, ssrc)) = self.ssrc_signalling.user_ssrc_map.remove(&id) {
|
||||
let _ = self.ssrc_signalling.ssrc_user_map.remove(&ssrc);
|
||||
|
||||
if let Some(state) = self.decoder_map.get_mut(&ssrc) {
|
||||
// don't cleanup immediately: leave for later cycle
|
||||
// this is key with reorder/jitter buffers where we may
|
||||
// still need to decode post disconnect for ~0.2s.
|
||||
state.prune_time = now + Duration::from_secs(1);
|
||||
state.disconnected = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// now remove all dead ssrcs.
|
||||
self.decoder_map.retain(|_, v| v.prune_time > now);
|
||||
self.held.prune(now);
|
||||
|
||||
cleanup_time = now + Duration::from_secs(5);
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Jarvis patch: an RTP packet of an SSRC that Discord has not named yet is held, and the held
|
||||
/// ones are processed before the first packet after the name came (see held.rs).
|
||||
async fn process_udp_message(&mut self, interconnect: &Interconnect, packet: BytesMut) {
|
||||
if let demux::Demuxed::Rtp(rtp) = demux::demux(&packet) {
|
||||
let ssrc = rtp.get_ssrc();
|
||||
let now = Instant::now();
|
||||
if !self.ssrc_signalling.ssrc_user_map.contains_key(&ssrc) {
|
||||
self.held.hold(ssrc, now, packet);
|
||||
return;
|
||||
}
|
||||
for held in self.held.take(ssrc, now) {
|
||||
self.process_packet(interconnect, held).await;
|
||||
}
|
||||
}
|
||||
self.process_packet(interconnect, packet).await;
|
||||
}
|
||||
|
||||
/// Jarvis patch: processes the held packets of every SSRC that has been named since (held.rs).
|
||||
async fn release_named(&mut self, interconnect: &Interconnect) {
|
||||
let named: Vec<u32> = self
|
||||
.held
|
||||
.ssrcs()
|
||||
.filter(|ssrc| self.ssrc_signalling.ssrc_user_map.contains_key(ssrc))
|
||||
.collect();
|
||||
let now = Instant::now();
|
||||
for ssrc in named {
|
||||
for held in self.held.take(ssrc, now) {
|
||||
self.process_packet(interconnect, held).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn process_packet(&mut self, interconnect: &Interconnect, mut packet: BytesMut) {
|
||||
// NOTE: errors here (and in general for UDP) are not fatal to the connection.
|
||||
// Panics should be avoided due to adversarial nature of rx'd packets,
|
||||
// but correct handling should not prompt a reconnect.
|
||||
//
|
||||
// For simplicity, if the event task fails then we nominate the mixing thread
|
||||
// to rebuild their context etc. (hence, the `let _ =` statements.), as it will
|
||||
// try to make contact every 20ms.
|
||||
let crypto_mode = self.crypto_mode;
|
||||
|
||||
match demux::demux_mut(packet.as_mut()) {
|
||||
DemuxedMut::Rtp(mut rtp) => {
|
||||
if !rtp_valid(&rtp.to_immutable()) {
|
||||
error!("Illegal RTP message received.");
|
||||
return;
|
||||
}
|
||||
|
||||
let mut packet_data = if self.config.decode_mode.should_decrypt() {
|
||||
let out = self.cipher.decrypt_rtp_in_place(&mut rtp).map(|(s, t)| {
|
||||
if rtp.get_padding() != 0 {
|
||||
let payload = rtp.payload();
|
||||
let payload_length = payload.len();
|
||||
let padding_count = payload[payload_length - t - 1] as usize;
|
||||
|
||||
(s, t + padding_count, true)
|
||||
} else {
|
||||
(s, t, true)
|
||||
}
|
||||
});
|
||||
|
||||
if let Err(ref e) = out {
|
||||
warn!("RTP decryption failed: {:?}", e);
|
||||
}
|
||||
|
||||
out.ok()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if let Some((rtp_body_start, rtp_body_tail, decrypted)) = packet_data {
|
||||
let ssrc = rtp.get_ssrc();
|
||||
let payload = rtp.payload_mut();
|
||||
let payload_length = payload.len();
|
||||
let body = &mut payload[rtp_body_start..payload_length - rtp_body_tail];
|
||||
let body_length = body.len();
|
||||
|
||||
// If the packet is transport-decrypted, DAVE is enabled, and the packet
|
||||
// looks encrypted: https://daveprotocol.com/#protocol-frame-check
|
||||
// The packet must be at least 11 bytes consisting of:
|
||||
// - 8 byte truncated AES128-GCM authentication tag
|
||||
// - 1 byte protocol supplemental data size
|
||||
// - 2 byte magic marker
|
||||
if decrypted
|
||||
&& self.dave_protocol_version.load(Ordering::Relaxed) != 0
|
||||
&& body_length >= 11
|
||||
&& body[body_length - DAVE_MAGIC_MARKER.len()..body_length]
|
||||
== DAVE_MAGIC_MARKER
|
||||
{
|
||||
// Silently drop encrypted packets if it's not possible to decrypt them for
|
||||
// one reason or another; otherwise there'd be error logs for trying to decode
|
||||
// Opus
|
||||
let Some(user_id) = self.ssrc_signalling.ssrc_user_map.get(&ssrc) else {
|
||||
return;
|
||||
};
|
||||
let Some(ref mut dave_session) = *self.dave_session.write().unwrap() else {
|
||||
return;
|
||||
};
|
||||
|
||||
if !dave_session.is_ready() {
|
||||
return;
|
||||
}
|
||||
|
||||
let result = dave_session.decrypt(user_id.0, davey::MediaType::AUDIO, body);
|
||||
|
||||
match result {
|
||||
Ok(decrypted_body) => {
|
||||
packet_data = Some((
|
||||
rtp_body_start,
|
||||
rtp_body_tail + (body.len() - decrypted_body.len()),
|
||||
decrypted,
|
||||
));
|
||||
body[..decrypted_body.len()].copy_from_slice(&decrypted_body);
|
||||
},
|
||||
Err(davey::errors::DecryptError::NoDecryptorForUser)
|
||||
| Err(davey::errors::DecryptError::DecryptionFailed(
|
||||
davey::errors::DecryptorDecryptError::NoValidCryptorFound {
|
||||
..
|
||||
},
|
||||
)) => {
|
||||
// Silently drop encrypted packets for users whose ratchets are not configured yet.
|
||||
return;
|
||||
},
|
||||
Err(e) => {
|
||||
error!(error = ?e, "DAVE decryption failed");
|
||||
return;
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let rtp = rtp.to_immutable();
|
||||
let (rtp_body_start, rtp_body_tail, decrypted) = packet_data.unwrap_or_else(|| {
|
||||
(
|
||||
crypto_mode.payload_prefix_len(),
|
||||
crypto_mode.payload_suffix_len(),
|
||||
false,
|
||||
)
|
||||
});
|
||||
|
||||
let entry = self
|
||||
.decoder_map
|
||||
.entry(rtp.get_ssrc())
|
||||
.or_insert_with(|| SsrcState::new(&rtp, crypto_mode, &self.config));
|
||||
|
||||
// Only do this on RTP, rather than RTCP -- this pins decoder state liveness
|
||||
// to *speech* rather than just presence.
|
||||
entry.refresh_timer(self.config.decode_state_timeout.into());
|
||||
|
||||
let store_pkt = StoredPacket {
|
||||
packet: packet.freeze(),
|
||||
decrypted,
|
||||
};
|
||||
let packet = store_pkt.packet.clone();
|
||||
entry.store_packet(store_pkt, &self.config);
|
||||
|
||||
drop(interconnect.events.send(EventMessage::FireCoreEvent(
|
||||
CoreContext::RtpPacket(InternalRtpPacket {
|
||||
packet,
|
||||
payload_offset: rtp_body_start,
|
||||
payload_end_pad: rtp_body_tail,
|
||||
}),
|
||||
)));
|
||||
},
|
||||
DemuxedMut::Rtcp(mut rtcp) => {
|
||||
let packet_data = if self.config.decode_mode.should_decrypt() {
|
||||
let out = self.cipher.decrypt_rtcp_in_place(&mut rtcp);
|
||||
|
||||
if let Err(ref e) = out {
|
||||
warn!("RTCP decryption failed: {:?}", e);
|
||||
}
|
||||
|
||||
out.ok()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let (start, tail) = packet_data.unwrap_or_else(|| {
|
||||
(
|
||||
crypto_mode.payload_prefix_len(),
|
||||
crypto_mode.payload_suffix_len(),
|
||||
)
|
||||
});
|
||||
|
||||
drop(interconnect.events.send(EventMessage::FireCoreEvent(
|
||||
CoreContext::RtcpPacket(InternalRtcpPacket {
|
||||
packet: packet.freeze(),
|
||||
payload_offset: start,
|
||||
payload_end_pad: tail,
|
||||
}),
|
||||
)));
|
||||
},
|
||||
DemuxedMut::FailedParse(t) => {
|
||||
warn!("Failed to parse message of type {:?}.", t);
|
||||
},
|
||||
DemuxedMut::TooSmall => {
|
||||
warn!("Illegal UDP packet from voice server.");
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(skip(interconnect, rx, cipher))]
|
||||
pub(crate) async fn runner(
|
||||
mut interconnect: Interconnect,
|
||||
rx: Receiver<UdpRxMessage>,
|
||||
cipher: Cipher,
|
||||
crypto_mode: CryptoMode,
|
||||
config: Config,
|
||||
udp_socket: UdpSocket,
|
||||
ssrc_signalling: Arc<SsrcTracker>,
|
||||
dave_session: Arc<RwLock<Option<davey::DaveSession>>>,
|
||||
dave_protocol_version: Arc<AtomicU16>,
|
||||
) {
|
||||
trace!("UDP receive handle started.");
|
||||
|
||||
let mut state = UdpRx {
|
||||
cipher,
|
||||
crypto_mode,
|
||||
decoder_map: HashMap::new(),
|
||||
config,
|
||||
rx,
|
||||
ssrc_signalling,
|
||||
udp_socket,
|
||||
dave_session,
|
||||
dave_protocol_version,
|
||||
held: Held::default(),
|
||||
};
|
||||
|
||||
state.run(&mut interconnect).await;
|
||||
|
||||
trace!("UDP receive handle stopped.");
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn rtp_valid(packet: &RtpPacket<'_>) -> bool {
|
||||
packet.get_version() == RTP_VERSION && packet.get_payload_type() == RTP_PROFILE_TYPE
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod jarvis_held_tests {
|
||||
//! Jarvis patch (held.rs): the first packets of a speaker are not lost to a late Speaking op.
|
||||
use super::*;
|
||||
use crate::driver::DecodeMode;
|
||||
use discortp::rtp::MutableRtpPacket;
|
||||
use serenity_voice_model::id::UserId;
|
||||
|
||||
fn rtp(seq: u16, ssrc: u32) -> BytesMut {
|
||||
let mut buf = BytesMut::zeroed(MutableRtpPacket::minimum_packet_size() + 3);
|
||||
let mut p = MutableRtpPacket::new(&mut buf[..]).unwrap();
|
||||
p.set_version(RTP_VERSION);
|
||||
p.set_payload_type(RTP_PROFILE_TYPE);
|
||||
p.set_sequence(seq.into());
|
||||
p.set_ssrc(ssrc);
|
||||
buf
|
||||
}
|
||||
|
||||
fn sequences(events: &Receiver<EventMessage>) -> Vec<u16> {
|
||||
events
|
||||
.try_iter()
|
||||
.filter_map(|e| match e {
|
||||
EventMessage::FireCoreEvent(CoreContext::RtpPacket(p)) =>
|
||||
Some(RtpPacket::new(&p.packet).unwrap().get_sequence().0 .0),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn udp_rx() -> UdpRx {
|
||||
let mode = CryptoMode::Aes256Gcm;
|
||||
UdpRx {
|
||||
cipher: mode.cipher_from_key(&[0u8; 32]).unwrap(),
|
||||
crypto_mode: mode,
|
||||
decoder_map: HashMap::new(),
|
||||
config: Config::default().decode_mode(DecodeMode::Pass),
|
||||
rx: flume::unbounded().1,
|
||||
ssrc_signalling: Arc::new(SsrcTracker::default()),
|
||||
udp_socket: UdpSocket::bind("127.0.0.1:0").await.unwrap(),
|
||||
dave_session: Arc::new(RwLock::new(None)),
|
||||
dave_protocol_version: Arc::new(AtomicU16::new(0)),
|
||||
held: Held::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_short_first_utterance_is_released_when_named_without_a_later_packet() {
|
||||
let (core, _core_rx) = flume::unbounded();
|
||||
let (events, events_rx) = flume::unbounded();
|
||||
let (mixer, _mixer_rx) = flume::unbounded();
|
||||
let ic = Interconnect { core, events, mixer };
|
||||
let mut rx = udp_rx().await;
|
||||
|
||||
rx.process_udp_message(&ic, rtp(1, 42)).await;
|
||||
rx.process_udp_message(&ic, rtp(2, 42)).await;
|
||||
rx.release_named(&ic).await;
|
||||
assert!(sequences(&events_rx).is_empty(), "still unnamed");
|
||||
|
||||
// The Speaking op comes after the last packet of the utterance.
|
||||
rx.ssrc_signalling.ssrc_user_map.insert(42, UserId(7));
|
||||
rx.release_named(&ic).await;
|
||||
assert_eq!(sequences(&events_rx), vec![1, 2]);
|
||||
rx.release_named(&ic).await;
|
||||
assert!(sequences(&events_rx).is_empty(), "released once");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn packets_before_the_speaking_op_reach_handlers_once_it_names_the_ssrc() {
|
||||
let (core, _core_rx) = flume::unbounded();
|
||||
let (events, events_rx) = flume::unbounded();
|
||||
let (mixer, _mixer_rx) = flume::unbounded();
|
||||
let ic = Interconnect { core, events, mixer };
|
||||
let mut rx = udp_rx().await;
|
||||
|
||||
rx.process_udp_message(&ic, rtp(1, 42)).await;
|
||||
rx.process_udp_message(&ic, rtp(2, 42)).await;
|
||||
assert!(sequences(&events_rx).is_empty(), "held until the SSRC is named");
|
||||
|
||||
rx.ssrc_signalling.ssrc_user_map.insert(42, UserId(7));
|
||||
rx.process_udp_message(&ic, rtp(3, 42)).await;
|
||||
assert_eq!(sequences(&events_rx), vec![1, 2, 3]);
|
||||
|
||||
rx.process_udp_message(&ic, rtp(4, 42)).await;
|
||||
assert_eq!(sequences(&events_rx), vec![4], "a held packet is processed once");
|
||||
}
|
||||
}
|
||||
@@ -1,204 +0,0 @@
|
||||
use super::*;
|
||||
use bytes::Bytes;
|
||||
use std::collections::VecDeque;
|
||||
use tracing::trace;
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct StoredPacket {
|
||||
pub packet: Bytes,
|
||||
// We need to store this as it's possible that a user can change config modes.
|
||||
pub decrypted: bool,
|
||||
}
|
||||
|
||||
/// Determines whether an SSRC's packets should be decoded.
|
||||
///
|
||||
/// Playout requires us to keep an almost constant delay, to do so we build
|
||||
/// a user's packet buffer up to the required length ([`Config::playout_buffer_length`])
|
||||
/// ([`Self::Fill`]) and then emit packets on each tick ([`Self::Drain`]).
|
||||
///
|
||||
/// This gets a bit harder to reason about when users stop speaking. If a speech gap
|
||||
/// lasts longer than the playout buffer, then we can simply swap from `Drain` -> `Fill`.
|
||||
/// However, a genuine gap of `n` frames must lead to us reverting to `Fill` for `n` frames.
|
||||
/// To compute this, we use the RTP timestamp of two `seq`-adjacent packets at playout: if the next
|
||||
/// timestamp is too large, then we revert to `Fill`.
|
||||
///
|
||||
/// Small playout bursts also require care.
|
||||
///
|
||||
/// If timestamp info is incorrect, then in the worst case we eventually need to rebuffer if the delay
|
||||
/// drains to zero.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum PlayoutMode {
|
||||
Fill,
|
||||
Drain,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub enum PacketLookup {
|
||||
Packet(StoredPacket),
|
||||
MissedPacket,
|
||||
Filling,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct PlayoutBuffer {
|
||||
buffer: VecDeque<Option<StoredPacket>>,
|
||||
playout_mode: PlayoutMode,
|
||||
next_seq: RtpSequence,
|
||||
current_timestamp: Option<RtpTimestamp>,
|
||||
consecutive_store_fails: usize,
|
||||
}
|
||||
|
||||
impl PlayoutBuffer {
|
||||
pub fn new(capacity: usize, next_seq: RtpSequence) -> Self {
|
||||
Self {
|
||||
buffer: VecDeque::with_capacity(capacity),
|
||||
playout_mode: PlayoutMode::Fill,
|
||||
next_seq,
|
||||
current_timestamp: None,
|
||||
consecutive_store_fails: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Slot a received RTP packet into the correct location in the playout buffer using
|
||||
/// its sequence number, subject to maximums.
|
||||
///
|
||||
/// An out of bounds packet must create any remaining `None`s
|
||||
pub fn store_packet(&mut self, packet: StoredPacket, config: &Config) {
|
||||
let rtp = RtpPacket::new(&packet.packet)
|
||||
.expect("FATAL: earlier valid packet now invalid (store)");
|
||||
|
||||
if self.current_timestamp.is_none() {
|
||||
self.current_timestamp = Some(reset_timeout(&rtp, config));
|
||||
}
|
||||
|
||||
// compute index by taking wrapping difference between both seq numbers.
|
||||
// If the difference is *too big*, or in the past [also too big, in a way],
|
||||
// ignore the packet
|
||||
let pkt_seq = rtp.get_sequence().0;
|
||||
let desired_index = (pkt_seq - self.next_seq).0 as i16;
|
||||
|
||||
// Similar concept to fetch_packet -- if there's a critical desync, and we're unwilling
|
||||
// to slot this packet into an empty/stuck buffer then behave as though this packet is the next
|
||||
// sequence number we're releasing.
|
||||
let err_threshold = i16::from(config.playout_buffer_length.get() * 5);
|
||||
let handling_desync = (self.buffer.is_empty()
|
||||
|| self.consecutive_store_fails >= (err_threshold as usize))
|
||||
&& desired_index >= err_threshold;
|
||||
|
||||
if desired_index < 0 {
|
||||
trace!("Missed packet arrived late, discarding from playout.");
|
||||
} else if !handling_desync && desired_index >= 64 {
|
||||
trace!(
|
||||
"Packet arrived beyond playout max length({}): wanted slot {desired_index}.\
|
||||
ts {}, seq {}, next_out_seq {}",
|
||||
rtp.get_ssrc(),
|
||||
rtp.get_timestamp().0,
|
||||
rtp.get_sequence().0,
|
||||
self.next_seq,
|
||||
);
|
||||
self.consecutive_store_fails += 1;
|
||||
} else {
|
||||
let index = if handling_desync {
|
||||
self.buffer.clear();
|
||||
self.next_seq = pkt_seq;
|
||||
|
||||
0
|
||||
} else {
|
||||
desired_index as usize
|
||||
};
|
||||
while self.buffer.len() <= index {
|
||||
self.buffer.push_back(None);
|
||||
}
|
||||
self.buffer[index] = Some(packet);
|
||||
self.consecutive_store_fails = 0;
|
||||
}
|
||||
|
||||
if self.buffer.len() >= usize::from(config.playout_buffer_length.get()) {
|
||||
self.playout_mode = PlayoutMode::Drain;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn fetch_packet(&mut self, config: &Config) -> PacketLookup {
|
||||
if self.playout_mode == PlayoutMode::Fill {
|
||||
return PacketLookup::Filling;
|
||||
}
|
||||
|
||||
let out = match self.buffer.pop_front() {
|
||||
Some(Some(pkt)) => {
|
||||
let rtp = RtpPacket::new(&pkt.packet)
|
||||
.expect("FATAL: earlier valid packet now invalid (fetch)");
|
||||
|
||||
// The curr_ts captures the current playout point; we want to
|
||||
// be able to emit *all* packets with a smaller timestamp.
|
||||
// However, we need to handle this in a wrap-safe way.
|
||||
// ts_diff shows where the current time lies if we treat packet_ts
|
||||
// as 0, s.t. ts_diff >= 0 (equiv) packet_time <= curr_time.
|
||||
let curr_ts = self.current_timestamp.as_mut().unwrap();
|
||||
let pkt_ts = rtp.get_timestamp().0;
|
||||
let ts_diff = (*curr_ts - pkt_ts).0 as i32;
|
||||
|
||||
// At least one client in the wild has seen unusual timestamp behaviour: the
|
||||
// first packet sent out in a run of audio may have an older timestamp.
|
||||
// This could be badly timestamped, or could conceivably be an orphaned packet
|
||||
// from a prior run, or e.g.:
|
||||
// (n x RTP) -> [>100ms delay] -> (RTP) -> [long O(s) delay] -> (m x RTP)
|
||||
// This leaves us with two adjacent packets in the same playout with wildly varying
|
||||
// timestamps. We have a slightly tricky situation -- we need to preserve accurate
|
||||
// timing to correctly drain/refill/recreate very small pauses in audio, but don't
|
||||
// want to block indefinitely.
|
||||
//
|
||||
// We have a compromise here -- if an adjacent (Drain) packet has a ts gap
|
||||
// larger than it would take to go through multiple Fill/Drain cycles, then
|
||||
// treat its TS as the next expected value to avoid jamming the buffer and losing
|
||||
// later audio.
|
||||
let skip_after = i32::try_from(
|
||||
usize::from(config.playout_buffer_length.get()) * 5 * MONO_FRAME_SIZE,
|
||||
)
|
||||
.unwrap_or((AUDIO_FRAME_RATE * 2 * MONO_FRAME_SIZE) as i32);
|
||||
|
||||
if ts_diff >= 0 {
|
||||
// At or before expected timestamp.
|
||||
self.next_seq = (rtp.get_sequence() + 1).0;
|
||||
|
||||
PacketLookup::Packet(pkt)
|
||||
} else if ts_diff <= -skip_after {
|
||||
// >5 playouts ahead.
|
||||
self.next_seq = (rtp.get_sequence() + 1).0;
|
||||
*curr_ts = pkt_ts;
|
||||
PacketLookup::Packet(pkt)
|
||||
} else {
|
||||
trace!("Witholding packet: ts_diff is {ts_diff}");
|
||||
self.buffer.push_front(Some(pkt));
|
||||
self.playout_mode = PlayoutMode::Fill;
|
||||
PacketLookup::Filling
|
||||
}
|
||||
},
|
||||
Some(None) => {
|
||||
self.next_seq += 1;
|
||||
PacketLookup::MissedPacket
|
||||
},
|
||||
None => PacketLookup::Filling,
|
||||
};
|
||||
|
||||
if self.buffer.is_empty() {
|
||||
self.playout_mode = PlayoutMode::Fill;
|
||||
self.current_timestamp = None;
|
||||
}
|
||||
|
||||
if let Some(ts) = self.current_timestamp.as_mut() {
|
||||
*ts += &(MONO_FRAME_SIZE as u32);
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
pub fn next_seq(&self) -> RtpSequence {
|
||||
self.next_seq
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn reset_timeout(packet: &RtpPacket<'_>, config: &Config) -> RtpTimestamp {
|
||||
let t_shift = MONO_FRAME_SIZE * usize::from(config.playout_buffer_length.get());
|
||||
(packet.get_timestamp() + (t_shift as u32)).0
|
||||
}
|
||||
@@ -1,193 +0,0 @@
|
||||
use super::*;
|
||||
use crate::{
|
||||
driver::{
|
||||
tasks::error::{Error, Result},
|
||||
Channels,
|
||||
DecodeConfig,
|
||||
DecodeMode,
|
||||
},
|
||||
events::context_data::{RtpData, VoiceData},
|
||||
};
|
||||
use discortp::{rtp::RtpExtensionPacket, Packet, PacketSize};
|
||||
use opus2::{Decoder as OpusDecoder, ErrorCode};
|
||||
use tracing::{error, warn};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct SsrcState {
|
||||
playout_buffer: PlayoutBuffer,
|
||||
crypto_mode: CryptoMode,
|
||||
decoder: OpusDecoder,
|
||||
decode_size: PacketDecodeSize,
|
||||
pub(crate) prune_time: Instant,
|
||||
pub(crate) disconnected: bool,
|
||||
channels: Channels,
|
||||
}
|
||||
|
||||
impl SsrcState {
|
||||
pub fn new(pkt: &RtpPacket<'_>, crypto_mode: CryptoMode, config: &Config) -> Self {
|
||||
let playout_capacity = config.playout_buffer_length.get() + config.playout_spike_length;
|
||||
let (sample_rate, channels) = match config.decode_mode {
|
||||
DecodeMode::Decode(config) => (config.sample_rate, config.channels),
|
||||
DecodeMode::Decrypt | DecodeMode::Pass => Default::default(),
|
||||
};
|
||||
|
||||
Self {
|
||||
playout_buffer: PlayoutBuffer::new(usize::from(playout_capacity), pkt.get_sequence().0),
|
||||
crypto_mode,
|
||||
decoder: OpusDecoder::new(sample_rate.into(), channels.into())
|
||||
.expect("Failed to create new Opus decoder for source."),
|
||||
decode_size: PacketDecodeSize::TwentyMillis,
|
||||
prune_time: Instant::now() + config.decode_state_timeout.into(),
|
||||
disconnected: false,
|
||||
channels,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn reconfigure_decoder(&mut self, config: DecodeConfig) {
|
||||
self.decoder = OpusDecoder::new(config.sample_rate.into(), config.channels.into())
|
||||
.expect("Failed to create new Opus decoder for source.");
|
||||
|
||||
self.channels = config.channels;
|
||||
}
|
||||
|
||||
pub fn store_packet(&mut self, packet: StoredPacket, config: &Config) {
|
||||
self.playout_buffer.store_packet(packet, config);
|
||||
}
|
||||
|
||||
pub fn refresh_timer(&mut self, state_timeout: Duration) {
|
||||
if !self.disconnected {
|
||||
self.prune_time = Instant::now() + state_timeout;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_voice_tick(&mut self, config: &Config) -> Result<Option<VoiceData>> {
|
||||
// Acquire a packet from the playout buffer:
|
||||
// Update nexts, lasts...
|
||||
// different cases: null packet who we want to decode as a miss, and packet who we must ignore temporarily.
|
||||
let m_pkt = self.playout_buffer.fetch_packet(config);
|
||||
let pkt = match m_pkt {
|
||||
PacketLookup::Packet(StoredPacket { packet, decrypted }) => Some((packet, decrypted)),
|
||||
PacketLookup::MissedPacket => None,
|
||||
PacketLookup::Filling => return Ok(None),
|
||||
};
|
||||
|
||||
let mut out = VoiceData {
|
||||
packet: None,
|
||||
decoded_voice: None,
|
||||
};
|
||||
|
||||
let should_decode = config.decode_mode.should_decode();
|
||||
if let Some((packet, decrypted)) = pkt {
|
||||
let rtp = RtpPacket::new(&packet).unwrap();
|
||||
let extensions = rtp.get_extension() != 0;
|
||||
|
||||
let payload = rtp.payload();
|
||||
let payload_offset = self.crypto_mode.payload_prefix_len();
|
||||
let payload_end_pad = payload.len() - self.crypto_mode.payload_suffix_len();
|
||||
|
||||
// We still need to compute missed packets here in case of long loss chains or similar.
|
||||
// This occurs due to the fallback in 'store_packet' (i.e., empty buffer and massive seq difference).
|
||||
// Normal losses should be handled by the below `else` branch.
|
||||
let new_seq: u16 = rtp.get_sequence().into();
|
||||
let missed_packets = new_seq.saturating_sub(self.playout_buffer.next_seq().0);
|
||||
|
||||
// TODO: maybe hand over audio and extension indices alongside packet?
|
||||
let (audio, _packet_size) = self.scan_and_decode(
|
||||
&payload[payload_offset..payload_end_pad],
|
||||
extensions,
|
||||
missed_packets,
|
||||
should_decode && decrypted,
|
||||
)?;
|
||||
|
||||
let rtp_data = RtpData {
|
||||
packet,
|
||||
payload_offset,
|
||||
payload_end_pad,
|
||||
};
|
||||
|
||||
out.packet = Some(rtp_data);
|
||||
out.decoded_voice = audio;
|
||||
} else if should_decode {
|
||||
let mut audio = vec![0; self.decode_size.len()];
|
||||
let dest_samples = (&mut audio[..])
|
||||
.try_into()
|
||||
.expect("Decode logic will cap decode buffer size at i32::MAX.");
|
||||
let len = self.decoder.decode(&[], dest_samples, false)?;
|
||||
audio.truncate(2 * len);
|
||||
|
||||
out.decoded_voice = Some(audio);
|
||||
}
|
||||
|
||||
Ok(Some(out))
|
||||
}
|
||||
|
||||
fn scan_and_decode(
|
||||
&mut self,
|
||||
data: &[u8],
|
||||
extension: bool,
|
||||
missed_packets: u16,
|
||||
decode: bool,
|
||||
) -> Result<(Option<Vec<i16>>, usize)> {
|
||||
let start = if extension {
|
||||
RtpExtensionPacket::new(data)
|
||||
.map(|pkt| pkt.packet_size())
|
||||
.ok_or_else(|| {
|
||||
error!("Extension packet indicated, but insufficient space.");
|
||||
Error::IllegalVoicePacket
|
||||
})
|
||||
} else {
|
||||
Ok(0)
|
||||
}?;
|
||||
|
||||
let pkt = if decode {
|
||||
let mut out = vec![0; self.decode_size.len()];
|
||||
|
||||
for _ in 0..missed_packets {
|
||||
let dest_samples = (&mut out[..])
|
||||
.try_into()
|
||||
.expect("Decode logic will cap decode buffer size at i32::MAX.");
|
||||
if let Err(e) = self.decoder.decode(&[], dest_samples, false) {
|
||||
warn!("Issue while decoding for missed packet: {:?}.", e);
|
||||
}
|
||||
}
|
||||
|
||||
// In general, we should expect 20 ms frames.
|
||||
// However, Discord occasionally like to surprise us with something bigger.
|
||||
// This is *sender-dependent behaviour*.
|
||||
//
|
||||
// This should scan up to find the "correct" size that a source is using,
|
||||
// and then remember that.
|
||||
loop {
|
||||
let tried_audio_len = self.decoder.decode(&data[start..], &mut out, false);
|
||||
match tried_audio_len {
|
||||
Ok(audio_len) => {
|
||||
// Decoding to stereo: audio_len refers to sample count irrespective of channel count.
|
||||
// => multiply by number of channels.
|
||||
out.truncate(self.channels.channels() * audio_len);
|
||||
|
||||
break;
|
||||
},
|
||||
Err(e) if e.code() == ErrorCode::BufferTooSmall => {
|
||||
if self.decode_size.can_bump_up() {
|
||||
self.decode_size = self.decode_size.bump_up();
|
||||
out = vec![0; self.decode_size.len()];
|
||||
} else {
|
||||
error!("Received packet larger than Opus standard maximum,");
|
||||
return Err(Error::IllegalVoicePacket);
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
error!("Failed to decode received packet: {:?}.", e);
|
||||
return Err(e.into());
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
Some(out)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok((pkt, data.len() - start))
|
||||
}
|
||||
}
|
||||
559
app/vendor/songbird/src/driver/tasks/ws.rs
vendored
559
app/vendor/songbird/src/driver/tasks/ws.rs
vendored
@@ -1,559 +0,0 @@
|
||||
use super::message::*;
|
||||
use crate::{
|
||||
driver::tasks::error::DaveReinitError,
|
||||
events::CoreContext,
|
||||
model::{
|
||||
payload::{Heartbeat, Speaking},
|
||||
CloseCode as VoiceCloseCode,
|
||||
Event as GatewayEvent,
|
||||
FromPrimitive,
|
||||
SpeakingState,
|
||||
},
|
||||
ws::{Error as WsError, WsStream},
|
||||
ConnectionInfo,
|
||||
};
|
||||
use flume::Receiver;
|
||||
use rand::{distr::Uniform, Rng};
|
||||
use serenity_voice_model::{
|
||||
id::UserId,
|
||||
payload::{
|
||||
DaveMlsCommitWelcome,
|
||||
DaveMlsInvalidCommitWelcome,
|
||||
DaveMlsKeyPackage,
|
||||
DaveMlsProposalsOperationType,
|
||||
DaveTransitionReady,
|
||||
},
|
||||
};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
num::NonZeroU16,
|
||||
sync::{
|
||||
atomic::{AtomicU16, Ordering},
|
||||
Arc,
|
||||
RwLock,
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{
|
||||
select,
|
||||
time::{sleep_until, Instant},
|
||||
};
|
||||
#[cfg(feature = "tungstenite")]
|
||||
use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
|
||||
use tracing::{debug, info, instrument, trace, warn};
|
||||
|
||||
pub(crate) struct AuxNetwork {
|
||||
rx: Receiver<WsMessage>,
|
||||
ws_client: WsStream,
|
||||
dont_send: bool,
|
||||
|
||||
ssrc: u32,
|
||||
heartbeat_interval: Duration,
|
||||
|
||||
speaking: SpeakingState,
|
||||
last_heartbeat_nonce: Option<u64>,
|
||||
|
||||
attempt_idx: usize,
|
||||
info: ConnectionInfo,
|
||||
|
||||
dave_session: Arc<RwLock<Option<davey::DaveSession>>>,
|
||||
dave_protocol_version: Arc<AtomicU16>,
|
||||
dave_pending_transitions: HashMap<u16, u16>,
|
||||
recognized_user_ids: HashSet<UserId>,
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
ssrc_signalling: Arc<SsrcTracker>,
|
||||
}
|
||||
|
||||
impl AuxNetwork {
|
||||
pub(crate) fn new(
|
||||
evt_rx: Receiver<WsMessage>,
|
||||
ws_client: WsStream,
|
||||
ssrc: u32,
|
||||
heartbeat_interval: f64,
|
||||
attempt_idx: usize,
|
||||
info: ConnectionInfo,
|
||||
dave_session: Arc<RwLock<Option<davey::DaveSession>>>,
|
||||
dave_protocol_version: Arc<AtomicU16>,
|
||||
#[cfg(feature = "receive")] ssrc_signalling: Arc<SsrcTracker>,
|
||||
) -> Self {
|
||||
let mut recognized_user_ids = HashSet::new();
|
||||
|
||||
recognized_user_ids.insert(info.user_id.into());
|
||||
|
||||
Self {
|
||||
rx: evt_rx,
|
||||
ws_client,
|
||||
dont_send: false,
|
||||
|
||||
ssrc,
|
||||
heartbeat_interval: Duration::from_secs_f64(heartbeat_interval / 1000.0),
|
||||
|
||||
speaking: SpeakingState::empty(),
|
||||
last_heartbeat_nonce: None,
|
||||
|
||||
attempt_idx,
|
||||
info,
|
||||
|
||||
dave_session,
|
||||
dave_protocol_version,
|
||||
dave_pending_transitions: HashMap::new(),
|
||||
recognized_user_ids,
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
ssrc_signalling,
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
async fn run(&mut self, interconnect: &mut Interconnect) {
|
||||
let mut next_heartbeat = Instant::now() + self.heartbeat_interval;
|
||||
|
||||
loop {
|
||||
let mut ws_error = false;
|
||||
let mut should_reconnect = false;
|
||||
let mut ws_reason = None;
|
||||
|
||||
let hb = sleep_until(next_heartbeat);
|
||||
|
||||
select! {
|
||||
// Biased polling (polling from top to bottom) is needed to process WebSocket events
|
||||
// queued in the initial handshake (messages before SessionDescription). One of the
|
||||
// events queued is ClientsConnect, which is needed to correctly keep track of
|
||||
// recognized_user_ids and to correctly process DaveMlsProposals.
|
||||
biased;
|
||||
|
||||
() = hb => {
|
||||
ws_error = match self.send_heartbeat().await {
|
||||
Err(e) => {
|
||||
should_reconnect = ws_error_is_not_final(&e);
|
||||
ws_reason = Some((&e).into());
|
||||
true
|
||||
},
|
||||
_ => false,
|
||||
};
|
||||
next_heartbeat = self.next_heartbeat();
|
||||
}
|
||||
inner_msg = self.rx.recv_async() => {
|
||||
match inner_msg {
|
||||
Ok(WsMessage::Ws(data)) => {
|
||||
self.ws_client = *data;
|
||||
next_heartbeat = self.next_heartbeat();
|
||||
self.dont_send = false;
|
||||
},
|
||||
Ok(WsMessage::ReplaceInterconnect(i)) => {
|
||||
*interconnect = i;
|
||||
},
|
||||
Ok(WsMessage::SetKeepalive(keepalive)) => {
|
||||
self.heartbeat_interval = Duration::from_secs_f64(keepalive / 1000.0);
|
||||
next_heartbeat = self.next_heartbeat();
|
||||
},
|
||||
Ok(WsMessage::Speaking(is_speaking)) => {
|
||||
if self.speaking.contains(SpeakingState::MICROPHONE) != is_speaking && !self.dont_send {
|
||||
self.speaking.set(SpeakingState::MICROPHONE, is_speaking);
|
||||
info!("Changing to {:?}", self.speaking);
|
||||
|
||||
let ssu_status = self.ws_client
|
||||
.send_json(&GatewayEvent::from(Speaking {
|
||||
delay: Some(0),
|
||||
speaking: self.speaking,
|
||||
ssrc: self.ssrc,
|
||||
user_id: None,
|
||||
}))
|
||||
.await;
|
||||
|
||||
ws_error |= match ssu_status {
|
||||
Err(e) => {
|
||||
should_reconnect = ws_error_is_not_final(&e);
|
||||
ws_reason = Some((&e).into());
|
||||
true
|
||||
},
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
},
|
||||
Ok(WsMessage::Deliver(msg)) => {
|
||||
ws_error |= match self.process_ws(interconnect, msg).await {
|
||||
Err(e) => {
|
||||
should_reconnect = ws_error_is_not_final(&e);
|
||||
ws_reason = Some((&e).into());
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
},
|
||||
Err(flume::RecvError::Disconnected) => {
|
||||
break;
|
||||
},
|
||||
}
|
||||
}
|
||||
ws_msg = self.ws_client.recv_event_no_timeout(), if !self.dont_send => {
|
||||
ws_error = match ws_msg {
|
||||
Err(e) => {
|
||||
should_reconnect = ws_error_is_not_final(&e);
|
||||
ws_reason = Some((&e).into());
|
||||
true
|
||||
},
|
||||
Ok(Some(msg)) => {
|
||||
match self.process_ws(interconnect, msg).await {
|
||||
Err(e) => {
|
||||
should_reconnect = ws_error_is_not_final(&e);
|
||||
ws_reason = Some((&e).into());
|
||||
true
|
||||
},
|
||||
_ => false
|
||||
}
|
||||
},
|
||||
_ => false,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if ws_error {
|
||||
self.dont_send = true;
|
||||
|
||||
if should_reconnect {
|
||||
drop(interconnect.core.send(CoreMessage::Reconnect));
|
||||
} else {
|
||||
drop(interconnect.core.send(CoreMessage::SignalWsClosure(
|
||||
self.attempt_idx,
|
||||
self.info.clone(),
|
||||
ws_reason,
|
||||
)));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn next_heartbeat(&self) -> Instant {
|
||||
Instant::now() + self.heartbeat_interval
|
||||
}
|
||||
|
||||
async fn send_heartbeat(&mut self) -> Result<(), WsError> {
|
||||
// Discord have suddenly, mysteriously, started rejecting
|
||||
// ints-as-strings. Keep JS happy here, I suppose...
|
||||
const JS_MAX_INT: u64 = (1u64 << 53) - 1;
|
||||
let nonce_range =
|
||||
Uniform::new(0, JS_MAX_INT).expect("uniform range is finite and nonempty");
|
||||
let nonce = rand::rng().sample(nonce_range);
|
||||
self.last_heartbeat_nonce = Some(nonce);
|
||||
|
||||
trace!("Sent heartbeat {:?}", self.speaking);
|
||||
|
||||
if !self.dont_send {
|
||||
self.ws_client
|
||||
.send_json(&GatewayEvent::from(Heartbeat { nonce }))
|
||||
.await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn process_ws(
|
||||
&mut self,
|
||||
interconnect: &Interconnect,
|
||||
value: GatewayEvent,
|
||||
) -> Result<(), WsError> {
|
||||
match value {
|
||||
GatewayEvent::Speaking(ev) => {
|
||||
// Jarvis patch: the event goes out before the SSRC is named to the UDP task, so the
|
||||
// packets that task held for it (udp_rx/held.rs) reach event handlers after it.
|
||||
drop(interconnect.events.send(EventMessage::FireCoreEvent(
|
||||
CoreContext::SpeakingStateUpdate(ev),
|
||||
)));
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
if let Some(user_id) = &ev.user_id {
|
||||
self.ssrc_signalling.user_ssrc_map.insert(*user_id, ev.ssrc);
|
||||
self.ssrc_signalling.ssrc_user_map.insert(ev.ssrc, *user_id);
|
||||
}
|
||||
},
|
||||
GatewayEvent::ClientConnect(ev) => {
|
||||
debug!("Received discontinued ClientConnect: {:?}", ev);
|
||||
},
|
||||
GatewayEvent::ClientDisconnect(ev) => {
|
||||
#[cfg(feature = "receive")]
|
||||
{
|
||||
self.ssrc_signalling.disconnected_users.insert(ev.user_id);
|
||||
}
|
||||
|
||||
self.recognized_user_ids.remove(&ev.user_id);
|
||||
|
||||
drop(interconnect.events.send(EventMessage::FireCoreEvent(
|
||||
CoreContext::ClientDisconnect(ev),
|
||||
)));
|
||||
},
|
||||
GatewayEvent::ClientsConnect(ev) => {
|
||||
self.recognized_user_ids.extend(&ev.user_ids);
|
||||
},
|
||||
GatewayEvent::HeartbeatAck(ev) => {
|
||||
if let Some(nonce) = self.last_heartbeat_nonce.take() {
|
||||
if ev.nonce == nonce {
|
||||
trace!("Heartbeat ACK received.");
|
||||
} else {
|
||||
warn!(
|
||||
"Heartbeat nonce mismatch! Expected {}, saw {}.",
|
||||
nonce, ev.nonce
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
GatewayEvent::DavePrepareTransition(ev) => {
|
||||
self.dave_pending_transitions
|
||||
.insert(ev.transition_id, ev.protocol_version);
|
||||
|
||||
if ev.transition_id == 0 {
|
||||
self.execute_dave_transition(ev.transition_id);
|
||||
} else if ev.protocol_version == 0 {
|
||||
if let Some(ref mut dave_session) = *self.dave_session.write().unwrap() {
|
||||
dave_session.set_passthrough_mode(true, Some(120));
|
||||
}
|
||||
|
||||
self.ws_client
|
||||
.send_json(&GatewayEvent::from(DaveTransitionReady {
|
||||
transition_id: ev.transition_id,
|
||||
protocol_version: ev.protocol_version,
|
||||
}))
|
||||
.await?;
|
||||
}
|
||||
},
|
||||
GatewayEvent::DaveExecuteTransition(ev) => {
|
||||
self.execute_dave_transition(ev.transition_id);
|
||||
},
|
||||
GatewayEvent::DavePrepareEpoch(ev) if ev.epoch == 1 => {
|
||||
self.dave_protocol_version
|
||||
.store(ev.protocol_version, Ordering::Relaxed);
|
||||
match self.reinit_dave_session().await {
|
||||
Err(DaveReinitError::Ws(e)) => return Err(e),
|
||||
Err(e) => {
|
||||
warn!(error = ?e, "failed to reinitialize DAVE session");
|
||||
},
|
||||
_ => {},
|
||||
}
|
||||
},
|
||||
GatewayEvent::DaveMlsExternalSender(ev) => {
|
||||
if let Some(ref mut dave_session) = *self.dave_session.write().unwrap() {
|
||||
if let Err(e) = dave_session.set_external_sender(&ev.external_sender) {
|
||||
warn!(error = ?e, "error setting MLS external sender");
|
||||
}
|
||||
}
|
||||
},
|
||||
GatewayEvent::DaveMlsProposals(ev) => {
|
||||
let operation_type = match ev.operation_type {
|
||||
DaveMlsProposalsOperationType::Append => davey::ProposalsOperationType::APPEND,
|
||||
DaveMlsProposalsOperationType::Revoke => davey::ProposalsOperationType::REVOKE,
|
||||
};
|
||||
let result = if let Some(ref mut dave_session) = *self.dave_session.write().unwrap()
|
||||
{
|
||||
match dave_session.process_proposals(
|
||||
operation_type,
|
||||
&ev.proposals,
|
||||
Some(
|
||||
&self
|
||||
.recognized_user_ids
|
||||
.iter()
|
||||
.map(|u| u.0)
|
||||
.collect::<Vec<_>>(),
|
||||
),
|
||||
) {
|
||||
Ok(result) => result,
|
||||
Err(e) => {
|
||||
warn!(error = ?e, "error processing MLS proposals");
|
||||
None
|
||||
},
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if let Some(commit_welcome) = result {
|
||||
self.ws_client
|
||||
.send_binary(&GatewayEvent::from(DaveMlsCommitWelcome {
|
||||
commit: commit_welcome.commit,
|
||||
welcome: commit_welcome.welcome,
|
||||
}))
|
||||
.await?;
|
||||
}
|
||||
},
|
||||
GatewayEvent::DaveMlsAnnounceCommitTransition(ev) => {
|
||||
match self.dave_process_commit(&ev.commit_message) {
|
||||
Some(Ok(())) if ev.transition_id != 0 => {
|
||||
let protocol_version = self.dave_protocol_version.load(Ordering::Relaxed);
|
||||
|
||||
self.dave_pending_transitions
|
||||
.insert(ev.transition_id, protocol_version);
|
||||
self.ws_client
|
||||
.send_json(&GatewayEvent::from(DaveTransitionReady {
|
||||
transition_id: ev.transition_id,
|
||||
protocol_version,
|
||||
}))
|
||||
.await?;
|
||||
},
|
||||
Some(Err(e)) => {
|
||||
warn!("MLS commit errored: {e:?}");
|
||||
self.ws_client
|
||||
.send_json(&GatewayEvent::from(DaveMlsInvalidCommitWelcome {
|
||||
transition_id: ev.transition_id,
|
||||
}))
|
||||
.await?;
|
||||
match self.reinit_dave_session().await {
|
||||
Err(DaveReinitError::Ws(e)) => return Err(e),
|
||||
Err(e) => {
|
||||
warn!(error = ?e, "failed to reinitialize DAVE session");
|
||||
},
|
||||
_ => {},
|
||||
}
|
||||
},
|
||||
Some(Ok(())) | None => {},
|
||||
}
|
||||
},
|
||||
GatewayEvent::DaveMlsWelcome(ev) => match self.dave_process_welcome(&ev.welcome) {
|
||||
Some(Ok(())) if ev.transition_id != 0 => {
|
||||
let protocol_version = self.dave_protocol_version.load(Ordering::Relaxed);
|
||||
|
||||
self.dave_pending_transitions
|
||||
.insert(ev.transition_id, protocol_version);
|
||||
self.ws_client
|
||||
.send_json(&GatewayEvent::from(DaveTransitionReady {
|
||||
transition_id: ev.transition_id,
|
||||
protocol_version,
|
||||
}))
|
||||
.await?;
|
||||
},
|
||||
|
||||
Some(Err(e)) => {
|
||||
warn!("MLS welcome errored: {e:?}");
|
||||
self.ws_client
|
||||
.send_json(&GatewayEvent::from(DaveMlsInvalidCommitWelcome {
|
||||
transition_id: ev.transition_id,
|
||||
}))
|
||||
.await?;
|
||||
match self.reinit_dave_session().await {
|
||||
Err(DaveReinitError::Ws(e)) => return Err(e),
|
||||
Err(e) => {
|
||||
warn!(error = ?e, "failed to reinitialize DAVE session");
|
||||
},
|
||||
_ => {},
|
||||
}
|
||||
},
|
||||
Some(Ok(())) | None => {},
|
||||
},
|
||||
other => {
|
||||
trace!("Received other websocket data: {:?}", other);
|
||||
},
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn dave_process_commit(
|
||||
&mut self,
|
||||
commit_message: &[u8],
|
||||
) -> Option<Result<(), davey::errors::ProcessCommitError>> {
|
||||
let mut dave_session_lock = self.dave_session.write().unwrap();
|
||||
let dave_session = (*dave_session_lock).as_mut()?;
|
||||
|
||||
Some(dave_session.process_commit(commit_message))
|
||||
}
|
||||
|
||||
fn dave_process_welcome(
|
||||
&mut self,
|
||||
welcome: &[u8],
|
||||
) -> Option<Result<(), davey::errors::ProcessWelcomeError>> {
|
||||
let mut dave_session_lock = self.dave_session.write().unwrap();
|
||||
let dave_session = (*dave_session_lock).as_mut()?;
|
||||
|
||||
Some(dave_session.process_welcome(welcome))
|
||||
}
|
||||
|
||||
async fn reinit_dave_session(&mut self) -> Result<(), DaveReinitError> {
|
||||
let protocol_version = self.dave_protocol_version.load(Ordering::Relaxed);
|
||||
|
||||
if let Some(dave_protocol_version) = NonZeroU16::new(protocol_version) {
|
||||
let user_id = self.info.user_id.0.into();
|
||||
let channel_id = self.info.channel_id.0.into();
|
||||
|
||||
let key_package =
|
||||
if let Some(ref mut dave_session) = *self.dave_session.write().unwrap() {
|
||||
dave_session.reinit(dave_protocol_version, user_id, channel_id, None)?;
|
||||
dave_session.create_key_package()?
|
||||
} else {
|
||||
let mut dave_session =
|
||||
davey::DaveSession::new(dave_protocol_version, user_id, channel_id, None)?;
|
||||
let key_package = dave_session.create_key_package()?;
|
||||
|
||||
*self.dave_session.write().unwrap() = Some(dave_session);
|
||||
|
||||
key_package
|
||||
};
|
||||
|
||||
self.ws_client
|
||||
.send_binary(&GatewayEvent::DaveMlsKeyPackage(DaveMlsKeyPackage {
|
||||
key_package,
|
||||
}))
|
||||
.await?;
|
||||
} else if let Some(ref mut dave_session) = *self.dave_session.write().unwrap() {
|
||||
dave_session.reset()?;
|
||||
dave_session.set_passthrough_mode(true, Some(10));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn execute_dave_transition(&mut self, transition_id: u16) {
|
||||
let Some(new_version) = self.dave_pending_transitions.get(&transition_id).copied() else {
|
||||
warn!("Received DaveExecuteTransition for unknown transition ID {transition_id}");
|
||||
return;
|
||||
};
|
||||
let old_version = self.dave_protocol_version.load(Ordering::Relaxed);
|
||||
|
||||
self.dave_protocol_version
|
||||
.store(new_version, Ordering::Relaxed);
|
||||
|
||||
// Upgraded from transport-only encryption
|
||||
if transition_id > 0 && old_version == 0 && new_version != 0 {
|
||||
if let Some(ref mut dave_session) = *self.dave_session.write().unwrap() {
|
||||
dave_session.set_passthrough_mode(true, Some(10));
|
||||
}
|
||||
}
|
||||
|
||||
self.dave_pending_transitions.remove(&transition_id);
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(skip(interconnect, aux))]
|
||||
pub(crate) async fn runner(mut interconnect: Interconnect, mut aux: AuxNetwork) {
|
||||
trace!("WS thread started.");
|
||||
aux.run(&mut interconnect).await;
|
||||
trace!("WS thread finished.");
|
||||
}
|
||||
|
||||
fn ws_error_is_not_final(err: &WsError) -> bool {
|
||||
match err {
|
||||
#[cfg(feature = "tungstenite")]
|
||||
WsError::WsClosed(Some(frame)) => match frame.code {
|
||||
CloseCode::Library(l) =>
|
||||
if let Some(code) = VoiceCloseCode::from_u16(l) {
|
||||
code.should_resume()
|
||||
} else {
|
||||
true
|
||||
},
|
||||
_ => true,
|
||||
},
|
||||
#[cfg(feature = "tws")]
|
||||
WsError::WsClosed(Some(code)) => match (*code).into() {
|
||||
code @ 4000..=4999_u16 =>
|
||||
if let Some(code) = VoiceCloseCode::from_u16(code) {
|
||||
code.should_resume()
|
||||
} else {
|
||||
true
|
||||
},
|
||||
_ => true,
|
||||
},
|
||||
e => {
|
||||
debug!("Error sending/receiving ws {:?}.", e);
|
||||
true
|
||||
},
|
||||
}
|
||||
}
|
||||
267
app/vendor/songbird/src/driver/test_config.rs
vendored
267
app/vendor/songbird/src/driver/test_config.rs
vendored
@@ -1,267 +0,0 @@
|
||||
#![allow(missing_docs)]
|
||||
|
||||
use flume::{Receiver, Sender};
|
||||
|
||||
use crate::{
|
||||
tracks::{PlayMode, TrackHandle, TrackState},
|
||||
Event,
|
||||
EventContext,
|
||||
EventHandler,
|
||||
TrackEvent,
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum TickStyle {
|
||||
Timed,
|
||||
UntimedWithExecLimit(Receiver<u64>),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum OutputMessage {
|
||||
Passthrough(Vec<u8>),
|
||||
Mixed(Vec<f32>),
|
||||
Silent,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl OutputMessage {
|
||||
#[must_use]
|
||||
pub fn is_passthrough(&self) -> bool {
|
||||
matches!(self, Self::Passthrough(_))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn is_mixed(&self) -> bool {
|
||||
matches!(self, Self::Mixed(_))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn is_mixed_with_nonzero_signal(&self) -> bool {
|
||||
if let Self::Mixed(data) = self {
|
||||
data.iter().any(|v| *v != 0.0f32)
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn is_explicit_silence(&self) -> bool {
|
||||
*self == Self::Silent
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum OutputMode {
|
||||
Raw(Sender<TickMessage<OutputMessage>>),
|
||||
Rtp(Sender<TickMessage<Vec<u8>>>),
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum TickMessage<T> {
|
||||
El(T),
|
||||
NoEl,
|
||||
}
|
||||
|
||||
impl<T> From<T> for TickMessage<T> {
|
||||
fn from(val: T) -> Self {
|
||||
TickMessage::El(val)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<TickMessage<OutputMessage>> for OutputPacket {
|
||||
fn from(val: TickMessage<OutputMessage>) -> Self {
|
||||
match val {
|
||||
TickMessage::El(e) => OutputPacket::Raw(e),
|
||||
TickMessage::NoEl => OutputPacket::Empty,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<TickMessage<Vec<u8>>> for OutputPacket {
|
||||
fn from(val: TickMessage<Vec<u8>>) -> Self {
|
||||
match val {
|
||||
TickMessage::El(e) => OutputPacket::Rtp(e),
|
||||
TickMessage::NoEl => OutputPacket::Empty,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum OutputPacket {
|
||||
Raw(OutputMessage),
|
||||
Rtp(Vec<u8>),
|
||||
Empty,
|
||||
}
|
||||
|
||||
impl OutputPacket {
|
||||
#[must_use]
|
||||
pub fn raw(&self) -> Option<&OutputMessage> {
|
||||
if let Self::Raw(o) = self {
|
||||
Some(o)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum OutputReceiver {
|
||||
Raw(Receiver<TickMessage<OutputMessage>>),
|
||||
Rtp(Receiver<TickMessage<Vec<u8>>>),
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct DriverTestHandle {
|
||||
pub rx: OutputReceiver,
|
||||
pub tx: Sender<u64>,
|
||||
}
|
||||
|
||||
impl DriverTestHandle {
|
||||
#[must_use]
|
||||
pub fn recv(&self) -> OutputPacket {
|
||||
match &self.rx {
|
||||
OutputReceiver::Raw(rx) => rx.recv().unwrap().into(),
|
||||
OutputReceiver::Rtp(rx) => rx.recv().unwrap().into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn recv_async(&self) -> OutputPacket {
|
||||
match &self.rx {
|
||||
OutputReceiver::Raw(rx) => rx.recv_async().await.unwrap().into(),
|
||||
OutputReceiver::Rtp(rx) => rx.recv_async().await.unwrap().into(),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn len(&self) -> usize {
|
||||
match &self.rx {
|
||||
OutputReceiver::Raw(rx) => rx.len(),
|
||||
OutputReceiver::Rtp(rx) => rx.len(),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
|
||||
pub fn wait(&self, n_ticks: u64) {
|
||||
for _i in 0..n_ticks {
|
||||
drop(self.recv());
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn wait_async(&self, n_ticks: u64) {
|
||||
for _i in 0..n_ticks {
|
||||
drop(self.recv_async().await);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn spawn_ticker(&self) {
|
||||
let remote = self.clone();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
remote.skip(1).await;
|
||||
tokio::time::sleep(Duration::from_millis(1)).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub fn wait_noisy(&self, n_ticks: u64) {
|
||||
for _i in 0..n_ticks {
|
||||
match self.recv() {
|
||||
OutputPacket::Empty => eprintln!("pkt: Nothing"),
|
||||
OutputPacket::Rtp(p) => eprintln!("pkt: RTP[{}B]", p.len()),
|
||||
OutputPacket::Raw(OutputMessage::Silent) => eprintln!("pkt: Raw-Silent"),
|
||||
OutputPacket::Raw(OutputMessage::Passthrough(p)) => {
|
||||
eprintln!("pkt: Raw-Passthrough[{}B]", p.len());
|
||||
},
|
||||
OutputPacket::Raw(OutputMessage::Mixed(p)) => {
|
||||
eprintln!("pkt: Raw-Mixed[{}B]", p.len());
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn skip(&self, n_ticks: u64) {
|
||||
self.tick(n_ticks);
|
||||
self.wait_async(n_ticks).await;
|
||||
}
|
||||
|
||||
pub fn tick(&self, n_ticks: u64) {
|
||||
assert!(
|
||||
n_ticks != 0,
|
||||
"Number of ticks to advance driver/mixer must be >= 1."
|
||||
);
|
||||
|
||||
self.tx.send(n_ticks).unwrap();
|
||||
}
|
||||
|
||||
pub async fn ready_track(
|
||||
&self,
|
||||
handle: &TrackHandle,
|
||||
tick_wait: Option<Duration>,
|
||||
) -> TrackState {
|
||||
struct SongPlayable {
|
||||
tx: Sender<TrackState>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl EventHandler for SongPlayable {
|
||||
async fn act(&self, ctx: &crate::EventContext<'_>) -> Option<Event> {
|
||||
if let EventContext::Track(&[(state, _)]) = ctx {
|
||||
drop(self.tx.send(state.clone()));
|
||||
}
|
||||
|
||||
Some(Event::Cancel)
|
||||
}
|
||||
}
|
||||
|
||||
struct SongErred {
|
||||
tx: Sender<PlayMode>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl EventHandler for SongErred {
|
||||
async fn act(&self, ctx: &crate::EventContext<'_>) -> Option<Event> {
|
||||
if let EventContext::Track(&[(state, _)]) = ctx {
|
||||
drop(self.tx.send(state.playing.clone()));
|
||||
}
|
||||
|
||||
Some(Event::Cancel)
|
||||
}
|
||||
}
|
||||
|
||||
let (tx, rx) = flume::bounded(1);
|
||||
let (err_tx, err_rx) = flume::bounded(1);
|
||||
|
||||
handle
|
||||
.add_event(Event::Track(TrackEvent::Playable), SongPlayable { tx })
|
||||
.expect("Adding track evt should not fail before any ticks.");
|
||||
|
||||
handle
|
||||
.add_event(Event::Track(TrackEvent::Error), SongErred { tx: err_tx })
|
||||
.expect("Adding track evt should not fail before any ticks.");
|
||||
|
||||
loop {
|
||||
self.tick(1);
|
||||
tokio::time::sleep(tick_wait.unwrap_or_else(|| Duration::from_millis(20))).await;
|
||||
self.wait_async(1).await;
|
||||
|
||||
match err_rx.try_recv() {
|
||||
Ok(e) => panic!("Error reported on track: {e:?}"),
|
||||
Err(flume::TryRecvError::Empty | flume::TryRecvError::Disconnected) => {},
|
||||
}
|
||||
|
||||
match rx.try_recv() {
|
||||
Ok(val) => return val,
|
||||
Err(flume::TryRecvError::Disconnected) => panic!(),
|
||||
Err(flume::TryRecvError::Empty) => {},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
255
app/vendor/songbird/src/driver/test_impls.rs
vendored
255
app/vendor/songbird/src/driver/test_impls.rs
vendored
@@ -1,255 +0,0 @@
|
||||
#![allow(missing_docs)]
|
||||
|
||||
use super::{
|
||||
scheduler::*,
|
||||
tasks::{message::*, mixer::Mixer},
|
||||
*,
|
||||
};
|
||||
use crate::{
|
||||
constants::*,
|
||||
input::{
|
||||
cached::Compressed,
|
||||
codecs::{get_codec_registry, get_probe},
|
||||
RawAdapter,
|
||||
},
|
||||
test_utils,
|
||||
tracks::LoopState,
|
||||
};
|
||||
use flume::Receiver;
|
||||
use std::{
|
||||
io::Cursor,
|
||||
net::UdpSocket,
|
||||
num::NonZeroU16,
|
||||
sync::{atomic::AtomicU16, Arc, RwLock},
|
||||
};
|
||||
use tokio::runtime::Handle;
|
||||
|
||||
// create a dummied task + interconnect.
|
||||
// measure perf at varying numbers of sources (binary 1--64) without passthrough support.
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
pub type Listeners = (
|
||||
Receiver<CoreMessage>,
|
||||
Receiver<EventMessage>,
|
||||
Receiver<UdpRxMessage>,
|
||||
);
|
||||
|
||||
#[cfg(not(feature = "receive"))]
|
||||
pub type Listeners = (Receiver<CoreMessage>, Receiver<EventMessage>);
|
||||
|
||||
pub type DummyMixer = (Mixer, Listeners);
|
||||
|
||||
impl Mixer {
|
||||
#[must_use]
|
||||
pub fn mock(handle: Handle, softclip: bool) -> DummyMixer {
|
||||
let (mix_tx, mix_rx) = flume::unbounded();
|
||||
let (core_tx, core_rx) = flume::unbounded();
|
||||
let (event_tx, event_rx) = flume::unbounded();
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
let (udp_receiver_tx, udp_receiver_rx) = flume::unbounded();
|
||||
|
||||
let ic = Interconnect {
|
||||
core: core_tx,
|
||||
events: event_tx,
|
||||
mixer: mix_tx,
|
||||
};
|
||||
|
||||
// Scheduler must be created from a Tokio context...
|
||||
let (tx, rx) = flume::unbounded();
|
||||
handle.spawn_blocking(move || tx.send(crate::Config::default().use_softclip(softclip)));
|
||||
let config = rx.recv().unwrap();
|
||||
|
||||
let mut out = Mixer::new(mix_rx, handle, ic, config);
|
||||
|
||||
let udp_tx = UdpSocket::bind("0.0.0.0:0").expect("Failed to create send port.");
|
||||
udp_tx
|
||||
.connect("127.0.0.1:5316")
|
||||
.expect("Failed to connect to local dest port.");
|
||||
|
||||
let mode = CryptoMode::Aes256Gcm;
|
||||
let cipher = mode.cipher_from_key(&[0u8; 32]).unwrap();
|
||||
let crypto_state = mode.into();
|
||||
let dave_protocol_version = Arc::new(AtomicU16::new(davey::DAVE_PROTOCOL_VERSION));
|
||||
let dave_session = Arc::new(RwLock::new(Some(
|
||||
davey::DaveSession::new(
|
||||
NonZeroU16::new(1).expect("failed to initialize NonZeroU16 from static value"),
|
||||
1,
|
||||
1,
|
||||
None,
|
||||
)
|
||||
.expect("failed to initialize davey::DaveSession"),
|
||||
)));
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
let fake_conn = MixerConnection {
|
||||
cipher,
|
||||
crypto_state,
|
||||
dave_session,
|
||||
dave_protocol_version,
|
||||
udp_rx: udp_receiver_tx,
|
||||
udp_tx,
|
||||
};
|
||||
|
||||
#[cfg(not(feature = "receive"))]
|
||||
let fake_conn = MixerConnection {
|
||||
cipher,
|
||||
crypto_state,
|
||||
dave_session,
|
||||
dave_protocol_version,
|
||||
udp_tx,
|
||||
};
|
||||
|
||||
out.conn_active = Some(fake_conn);
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
return (out, (core_rx, event_rx, udp_receiver_rx));
|
||||
|
||||
#[cfg(not(feature = "receive"))]
|
||||
return (out, (core_rx, event_rx));
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn test_with_float(num_tracks: usize, handle: Handle, softclip: bool) -> DummyMixer {
|
||||
let mut out = Self::mock(handle, softclip);
|
||||
|
||||
let floats = test_utils::make_sine(10 * STEREO_FRAME_SIZE, true);
|
||||
|
||||
for _ in 0..num_tracks {
|
||||
let input: Input = RawAdapter::new(Cursor::new(floats.clone()), 48_000, 2).into();
|
||||
let promoted = match input {
|
||||
Input::Live(l, _) => l.promote(get_codec_registry(), get_probe()),
|
||||
Input::Lazy(_) => panic!("Failed to create a guaranteed source."),
|
||||
};
|
||||
let (_, ctx) = Track::from(Input::Live(promoted.unwrap(), None)).into_context();
|
||||
_ = out.0.add_track(ctx);
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn test_with_float_unending(handle: Handle, softclip: bool) -> (DummyMixer, TrackHandle) {
|
||||
let mut out = Self::mock(handle, softclip);
|
||||
|
||||
let floats = test_utils::make_sine(10 * STEREO_FRAME_SIZE, true);
|
||||
|
||||
let input: Input = RawAdapter::new(Cursor::new(floats.clone()), 48_000, 2).into();
|
||||
let promoted = match input {
|
||||
Input::Live(l, _) => l.promote(get_codec_registry(), get_probe()),
|
||||
Input::Lazy(_) => panic!("Failed to create a guaranteed source."),
|
||||
};
|
||||
let mut track = Track::from(Input::Live(promoted.unwrap(), None));
|
||||
track.loops = LoopState::Infinite;
|
||||
|
||||
let (handle, ctx) = track.into_context();
|
||||
_ = out.0.add_track(ctx);
|
||||
|
||||
(out, handle)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn test_with_float_drop(num_tracks: usize, handle: Handle) -> DummyMixer {
|
||||
let mut out = Self::mock(handle, true);
|
||||
|
||||
for i in 0..num_tracks {
|
||||
let floats = test_utils::make_sine((i / 5) * STEREO_FRAME_SIZE, true);
|
||||
let input: Input = RawAdapter::new(Cursor::new(floats.clone()), 48_000, 2).into();
|
||||
let promoted = match input {
|
||||
Input::Live(l, _) => l.promote(get_codec_registry(), get_probe()),
|
||||
Input::Lazy(_) => panic!("Failed to create a guaranteed source."),
|
||||
};
|
||||
let (_, ctx) = Track::from(Input::Live(promoted.unwrap(), None)).into_context();
|
||||
_ = out.0.add_track(ctx);
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn test_with_opus(handle: &Handle) -> DummyMixer {
|
||||
// should add a single opus-based track.
|
||||
// make this fully loaded to prevent any perf cost there.
|
||||
let mut out = Self::mock(handle.clone(), false);
|
||||
|
||||
let floats = test_utils::make_sine(6 * STEREO_FRAME_SIZE, true);
|
||||
|
||||
let input: Input = RawAdapter::new(Cursor::new(floats), 48_000, 2).into();
|
||||
|
||||
let mut src = handle.block_on(async move {
|
||||
Compressed::new(input, Bitrate::Bits(128_000))
|
||||
.await
|
||||
.expect("These parameters are well-defined.")
|
||||
});
|
||||
|
||||
src.raw.load_all();
|
||||
|
||||
let promoted = match src.into() {
|
||||
Input::Live(l, _) => l.promote(get_codec_registry(), get_probe()),
|
||||
Input::Lazy(_) => panic!("Failed to create a guaranteed source."),
|
||||
};
|
||||
let (_, ctx) = Track::from(Input::Live(promoted.unwrap(), None)).into_context();
|
||||
|
||||
_ = out.0.add_track(ctx);
|
||||
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
pub struct MockScheduler {
|
||||
pub core: Live,
|
||||
pub stats: Arc<StatBlock>,
|
||||
pub local: Arc<LiveStatBlock>,
|
||||
pub rx: Receiver<SchedulerMessage>,
|
||||
pub tx: Sender<(TaskId, ParkedMixer)>,
|
||||
pub id: TaskId,
|
||||
}
|
||||
|
||||
impl MockScheduler {
|
||||
#[must_use]
|
||||
pub fn new(mode: Option<Mode>) -> Self {
|
||||
let stats = Arc::new(StatBlock::default());
|
||||
let local = Arc::new(LiveStatBlock::default());
|
||||
|
||||
let (task_tx, task_rx) = flume::unbounded();
|
||||
let (sched_tx, sched_rx) = flume::unbounded();
|
||||
|
||||
let cfg = crate::driver::SchedulerConfig {
|
||||
strategy: mode.unwrap_or_default(),
|
||||
move_expensive_tasks: true,
|
||||
};
|
||||
|
||||
let core = Live::new(
|
||||
WorkerId::new(),
|
||||
cfg,
|
||||
stats.clone(),
|
||||
local.clone(),
|
||||
task_rx,
|
||||
sched_tx,
|
||||
);
|
||||
Self {
|
||||
core,
|
||||
stats,
|
||||
local,
|
||||
rx: sched_rx,
|
||||
tx: task_tx,
|
||||
id: TaskId::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_mixer_direct(&mut self, m: Mixer) {
|
||||
let id = self.id.incr();
|
||||
self.core.add_task_direct(m, id);
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn from_mixers(mode: Option<Mode>, mixers: Vec<DummyMixer>) -> (Self, Vec<Listeners>) {
|
||||
let mut out = Self::new(mode);
|
||||
let mut listeners = vec![];
|
||||
for (mixer, listener) in mixers {
|
||||
out.add_mixer_direct(mixer);
|
||||
listeners.push(listener);
|
||||
}
|
||||
(out, listeners)
|
||||
}
|
||||
}
|
||||
148
app/vendor/songbird/src/error.rs
vendored
148
app/vendor/songbird/src/error.rs
vendored
@@ -1,148 +0,0 @@
|
||||
//! Driver and gateway error handling.
|
||||
|
||||
#[cfg(feature = "serenity")]
|
||||
use futures::channel::mpsc::TrySendError;
|
||||
pub use serde_json::Error as JsonError;
|
||||
#[cfg(feature = "serenity")]
|
||||
use serenity::gateway::ShardRunnerMessage;
|
||||
#[cfg(feature = "gateway")]
|
||||
use std::{error::Error, fmt};
|
||||
#[cfg(feature = "twilight")]
|
||||
use twilight_gateway::error::ChannelError;
|
||||
|
||||
#[cfg(feature = "gateway")]
|
||||
#[derive(Debug)]
|
||||
#[non_exhaustive]
|
||||
/// Error returned when a manager or call handler is
|
||||
/// unable to send messages over Discord's gateway.
|
||||
pub enum JoinError {
|
||||
/// Request to join was dropped, cancelled, or replaced.
|
||||
Dropped,
|
||||
/// No available gateway connection was provided to send
|
||||
/// voice state update messages.
|
||||
NoSender,
|
||||
/// Tried to leave a [`Call`] which was not found.
|
||||
///
|
||||
/// [`Call`]: crate::Call
|
||||
NoCall,
|
||||
/// Connection details were not received from Discord in the
|
||||
/// time given in [the `Call`'s configuration].
|
||||
///
|
||||
/// This can occur if a message is lost by the Discord client
|
||||
/// between restarts, or if Discord's gateway believes that
|
||||
/// this bot is still in the channel it attempts to join.
|
||||
///
|
||||
/// *Users should `leave` the server on the gateway before
|
||||
/// re-attempting connection.*
|
||||
///
|
||||
/// [the `Call`'s configuration]: crate::Config
|
||||
TimedOut,
|
||||
#[cfg(feature = "driver")]
|
||||
/// The driver failed to establish a voice connection.
|
||||
///
|
||||
/// *Users should `leave` the server on the gateway before
|
||||
/// re-attempting connection.*
|
||||
Driver(ConnectionError),
|
||||
#[cfg(feature = "serenity")]
|
||||
/// Serenity-specific WebSocket send error.
|
||||
Serenity(Box<TrySendError<ShardRunnerMessage>>),
|
||||
#[cfg(feature = "twilight")]
|
||||
/// Twilight-specific WebSocket send error when a message fails to send over websocket.
|
||||
Twilight(ChannelError),
|
||||
}
|
||||
|
||||
#[cfg(feature = "gateway")]
|
||||
impl JoinError {
|
||||
/// Indicates whether this failure may have left (or been
|
||||
/// caused by) Discord's gateway state being in an
|
||||
/// inconsistent state.
|
||||
///
|
||||
/// Failure to `leave` before rejoining may cause further
|
||||
/// timeouts.
|
||||
pub fn should_leave_server(&self) -> bool {
|
||||
matches!(self, JoinError::TimedOut)
|
||||
}
|
||||
|
||||
#[cfg(feature = "driver")]
|
||||
/// Indicates whether this failure can be reattempted via
|
||||
/// [`Driver::connect`] with retreived connection info.
|
||||
///
|
||||
/// Failure to `leave` before rejoining may cause further
|
||||
/// timeouts.
|
||||
///
|
||||
/// [`Driver::connect`]: crate::driver::Driver
|
||||
pub fn should_reconnect_driver(&self) -> bool {
|
||||
matches!(self, JoinError::Driver(_))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "gateway")]
|
||||
impl fmt::Display for JoinError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "failed to join voice channel: ")?;
|
||||
match self {
|
||||
JoinError::Dropped => write!(f, "request was cancelled/dropped"),
|
||||
JoinError::NoSender => write!(f, "no gateway destination"),
|
||||
JoinError::NoCall => write!(f, "tried to leave a non-existent call"),
|
||||
JoinError::TimedOut => write!(f, "gateway response from Discord timed out"),
|
||||
#[cfg(feature = "driver")]
|
||||
JoinError::Driver(_) => write!(f, "establishing connection failed"),
|
||||
#[cfg(feature = "serenity")]
|
||||
JoinError::Serenity(e) => e.fmt(f),
|
||||
#[cfg(feature = "twilight")]
|
||||
JoinError::Twilight(e) => e.fmt(f),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "gateway")]
|
||||
impl Error for JoinError {
|
||||
fn source(&self) -> Option<&(dyn Error + 'static)> {
|
||||
match self {
|
||||
JoinError::Dropped => None,
|
||||
JoinError::NoSender => None,
|
||||
JoinError::NoCall => None,
|
||||
JoinError::TimedOut => None,
|
||||
#[cfg(feature = "driver")]
|
||||
JoinError::Driver(e) => Some(e),
|
||||
#[cfg(feature = "serenity")]
|
||||
JoinError::Serenity(e) => e.source(),
|
||||
#[cfg(feature = "twilight")]
|
||||
JoinError::Twilight(e) => e.source(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "serenity", feature = "gateway"))]
|
||||
impl From<Box<TrySendError<ShardRunnerMessage>>> for JoinError {
|
||||
fn from(e: Box<TrySendError<ShardRunnerMessage>>) -> Self {
|
||||
JoinError::Serenity(e)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "twilight", feature = "gateway"))]
|
||||
impl From<ChannelError> for JoinError {
|
||||
fn from(e: ChannelError) -> Self {
|
||||
JoinError::Twilight(e)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "driver", feature = "gateway"))]
|
||||
impl From<ConnectionError> for JoinError {
|
||||
fn from(e: ConnectionError) -> Self {
|
||||
JoinError::Driver(e)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "gateway")]
|
||||
/// Convenience type for Discord gateway error handling.
|
||||
pub type JoinResult<T> = Result<T, JoinError>;
|
||||
|
||||
#[cfg(feature = "driver")]
|
||||
pub use crate::{
|
||||
driver::{
|
||||
connection::error::{Error as ConnectionError, Result as ConnectionResult},
|
||||
SchedulerError,
|
||||
},
|
||||
tracks::{ControlError, PlayError, TrackResult},
|
||||
};
|
||||
@@ -1,31 +0,0 @@
|
||||
use crate::id::*;
|
||||
|
||||
/// Voice connection details gathered at setup/reinstantiation.
|
||||
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
|
||||
#[non_exhaustive]
|
||||
pub struct ConnectData<'a> {
|
||||
/// ID of the voice channel being joined.
|
||||
///
|
||||
/// This can be used to reconnect/renew a voice session via the gateway.
|
||||
pub channel_id: ChannelId,
|
||||
/// ID of the target voice channel's parent guild.
|
||||
pub guild_id: GuildId,
|
||||
/// Unique string describing this session for validation/authentication purposes.
|
||||
pub session_id: &'a str,
|
||||
/// The domain name of Discord's voice/TURN server.
|
||||
///
|
||||
/// With the introduction of Discord's automatic voice server selection,
|
||||
/// this is no longer guaranteed to match a server's settings. This field
|
||||
/// may be useful if you need/wish to move your voice connection to a node/shard
|
||||
/// closer to Discord.
|
||||
pub server: &'a str,
|
||||
/// The [RTP SSRC] *("Synchronisation source")* assigned by the voice server
|
||||
/// for the duration of this call.
|
||||
///
|
||||
/// All packets sent will use this SSRC, which is not related to the sender's User
|
||||
/// ID. These are usually allocated sequentially by Discord, following on from
|
||||
/// a random starting SSRC.
|
||||
///
|
||||
/// [RTP SSRC]: https://tools.ietf.org/html/rfc3550#section-3
|
||||
pub ssrc: u32,
|
||||
}
|
||||
@@ -1,126 +0,0 @@
|
||||
use crate::{
|
||||
error::ConnectionError,
|
||||
id::*,
|
||||
model::{CloseCode as VoiceCloseCode, FromPrimitive},
|
||||
ws::Error as WsError,
|
||||
};
|
||||
#[cfg(feature = "tungstenite")]
|
||||
use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
|
||||
|
||||
/// Voice connection details gathered at termination or failure.
|
||||
///
|
||||
/// In the event of a failure, this event data is gathered after
|
||||
/// a reconnection strategy has exhausted all of its attempts.
|
||||
#[derive(Debug)]
|
||||
#[non_exhaustive]
|
||||
pub struct DisconnectData<'a> {
|
||||
/// The location that a voice connection was terminated.
|
||||
pub kind: DisconnectKind,
|
||||
/// The cause of any connection failure.
|
||||
///
|
||||
/// If `None`, then this disconnect was requested by the user in some way
|
||||
/// (i.e., leaving or changing voice channels).
|
||||
pub reason: Option<DisconnectReason>,
|
||||
/// ID of the voice channel being joined.
|
||||
///
|
||||
/// This can be used to reconnect/renew a voice session via the gateway.
|
||||
pub channel_id: ChannelId,
|
||||
/// ID of the target voice channel's parent guild.
|
||||
pub guild_id: GuildId,
|
||||
/// Unique string describing this session for validation/authentication purposes.
|
||||
pub session_id: &'a str,
|
||||
}
|
||||
|
||||
/// The location that a voice connection was terminated.
|
||||
#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)]
|
||||
#[non_exhaustive]
|
||||
pub enum DisconnectKind {
|
||||
/// The voice driver failed to connect to the server.
|
||||
///
|
||||
/// This requires explicit handling at the gateway level
|
||||
/// to either reconnect or fully disconnect.
|
||||
Connect,
|
||||
/// The voice driver failed to reconnect to the server.
|
||||
///
|
||||
/// This requires explicit handling at the gateway level
|
||||
/// to either reconnect or fully disconnect.
|
||||
Reconnect,
|
||||
/// The voice connection was terminated mid-session by either
|
||||
/// the user or Discord.
|
||||
///
|
||||
/// If `reason == None`, then this disconnection is either
|
||||
/// a full disconnect or a user-requested channel change.
|
||||
/// Otherwise, this is likely a session expiry (requiring user
|
||||
/// handling to fully disconnect/reconnect).
|
||||
Runtime,
|
||||
}
|
||||
|
||||
/// The reason that a voice connection failed.
|
||||
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
|
||||
#[non_exhaustive]
|
||||
pub enum DisconnectReason {
|
||||
/// This (re)connection attempt was dropped due to another request.
|
||||
AttemptDiscarded,
|
||||
/// Songbird had an internal error.
|
||||
///
|
||||
/// This should never happen; if this is ever seen, raise an issue with logs.
|
||||
Internal,
|
||||
/// A host-specific I/O error caused the fault; this is likely transient, and
|
||||
/// should be retried some time later.
|
||||
Io,
|
||||
/// Songbird and Discord disagreed on the protocol used to establish a
|
||||
/// voice connection.
|
||||
///
|
||||
/// This should never happen; if this is ever seen, raise an issue with logs.
|
||||
ProtocolViolation,
|
||||
/// A voice connection was not established in the specified time.
|
||||
TimedOut,
|
||||
/// The call was manually disconnected by a user command, e.g. [`Driver::leave`].
|
||||
///
|
||||
/// [`Driver::leave`]: crate::driver::Driver::leave
|
||||
Requested,
|
||||
/// The Websocket connection was closed by Discord.
|
||||
///
|
||||
/// This typically indicates that the voice session has expired,
|
||||
/// and a new one needs to be requested via the gateway.
|
||||
WsClosed(Option<VoiceCloseCode>),
|
||||
}
|
||||
|
||||
impl From<&ConnectionError> for DisconnectReason {
|
||||
fn from(e: &ConnectionError) -> Self {
|
||||
match e {
|
||||
ConnectionError::AttemptDiscarded => Self::AttemptDiscarded,
|
||||
ConnectionError::CryptoInvalidLength
|
||||
| ConnectionError::CryptoModeInvalid
|
||||
| ConnectionError::CryptoModeUnavailable
|
||||
| ConnectionError::DaveCreateKeyPackageError(_)
|
||||
| ConnectionError::DaveInitializationError(_)
|
||||
| ConnectionError::EndpointUrl
|
||||
| ConnectionError::IllegalDiscoveryResponse
|
||||
| ConnectionError::IllegalIp
|
||||
| ConnectionError::Json(_) => Self::ProtocolViolation,
|
||||
ConnectionError::Io(_) => Self::Io,
|
||||
ConnectionError::Crypto(_) | ConnectionError::InterconnectFailure(_) => Self::Internal,
|
||||
ConnectionError::Ws(ws) => ws.into(),
|
||||
ConnectionError::TimedOut => Self::TimedOut,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&WsError> for DisconnectReason {
|
||||
fn from(e: &WsError) -> Self {
|
||||
Self::WsClosed(match e {
|
||||
#[cfg(feature = "tungstenite")]
|
||||
WsError::WsClosed(Some(frame)) => match frame.code {
|
||||
CloseCode::Library(l) => VoiceCloseCode::from_u16(l),
|
||||
_ => None,
|
||||
},
|
||||
#[cfg(feature = "tws")]
|
||||
WsError::WsClosed(Some(code)) => match (*code).into() {
|
||||
code @ 4000..=4999_u16 => VoiceCloseCode::from_u16(code),
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
//! Types containing the main body of an [`EventContext`].
|
||||
//!
|
||||
//! [`EventContext`]: super::EventContext
|
||||
mod connect;
|
||||
mod disconnect;
|
||||
#[cfg(feature = "receive")]
|
||||
mod rtcp;
|
||||
#[cfg(feature = "receive")]
|
||||
mod rtp;
|
||||
#[cfg(feature = "receive")]
|
||||
mod voice;
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
use bytes::Bytes;
|
||||
|
||||
pub use self::{connect::*, disconnect::*};
|
||||
#[cfg(feature = "receive")]
|
||||
pub use self::{rtcp::*, rtp::*, voice::*};
|
||||
@@ -1,29 +0,0 @@
|
||||
use discortp::rtcp::RtcpPacket;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
#[non_exhaustive]
|
||||
/// Telemetry/statistics packet, received from another stream
|
||||
///
|
||||
/// `payload_offset` contains the true payload location within the raw packet's `payload()`,
|
||||
/// to allow manual decoding of `Rtcp` packet bodies.
|
||||
pub struct RtcpData {
|
||||
/// Raw RTCP packet data.
|
||||
pub packet: Bytes,
|
||||
/// Byte index into the packet body (after headers) for where the payload begins.
|
||||
pub payload_offset: usize,
|
||||
/// Number of bytes at the end of the packet to discard.
|
||||
pub payload_end_pad: usize,
|
||||
}
|
||||
|
||||
impl RtcpData {
|
||||
/// Create a zero-copy view of the inner RTCP packet.
|
||||
///
|
||||
/// This allows easy access to packet header fields, taking them from the underlying
|
||||
/// `Bytes` as needed while handling endianness etc.
|
||||
pub fn rtcp(&'_ self) -> RtcpPacket<'_> {
|
||||
RtcpPacket::new(&self.packet)
|
||||
.expect("FATAL: leaked illegally small RTP packet from UDP Rx task.")
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
use discortp::rtp::RtpPacket;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
#[non_exhaustive]
|
||||
/// Opus audio packet, received from another stream
|
||||
///
|
||||
/// `payload_offset` contains the true payload location within the raw packet's `payload()`,
|
||||
/// if extensions or raw packet data are required.
|
||||
pub struct RtpData {
|
||||
/// Raw RTP packet data.
|
||||
///
|
||||
/// Includes the SSRC (i.e., sender) of this packet.
|
||||
pub packet: Bytes,
|
||||
/// Byte index into the packet body (after headers) for where the payload begins.
|
||||
pub payload_offset: usize,
|
||||
/// Number of bytes at the end of the packet to discard.
|
||||
pub payload_end_pad: usize,
|
||||
}
|
||||
|
||||
impl RtpData {
|
||||
/// Create a zero-copy view of the inner RTP packet.
|
||||
///
|
||||
/// This allows easy access to packet header fields, taking them from the underlying
|
||||
/// `Bytes` as needed while handling endianness etc.
|
||||
pub fn rtp(&'_ self) -> RtpPacket<'_> {
|
||||
RtpPacket::new(&self.packet)
|
||||
.expect("FATAL: leaked illegally small RTP packet from UDP Rx task.")
|
||||
}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
#[non_exhaustive]
|
||||
/// Audio data from all users in a voice channel, fired every 20ms.
|
||||
///
|
||||
/// Songbird implements a jitter buffer to sycnhronise user packets, smooth out network latency, and
|
||||
/// handle packet reordering by the network. Packet playout via this event is delayed by approximately
|
||||
/// [`Config::playout_buffer_length`]` * 20ms` from its original arrival.
|
||||
///
|
||||
/// [`Config::playout_buffer_length`]: crate::Config::playout_buffer_length
|
||||
pub struct VoiceTick {
|
||||
/// Decoded voice data and source packets sent by each user.
|
||||
pub speaking: HashMap<u32, VoiceData>,
|
||||
|
||||
/// Set of all SSRCs currently known in the call who aren't included in [`Self::speaking`].
|
||||
pub silent: HashSet<u32>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
#[non_exhaustive]
|
||||
/// Voice packet and audio data for a single user, from a single tick.
|
||||
pub struct VoiceData {
|
||||
/// RTP packet clocked out for this tick.
|
||||
///
|
||||
/// If `None`, then the packet was lost, and [`Self::decoded_voice`] may include
|
||||
/// around one codec delay's worth of audio.
|
||||
pub packet: Option<RtpData>,
|
||||
/// PCM audio obtained from a user.
|
||||
///
|
||||
/// Valid audio data (`Some(audio)` where `audio.len >= 0`) typically contains 20ms of 16-bit PCM audio
|
||||
/// using native endianness. This defaults to stereo audio at 48kHz, and can be configured via
|
||||
/// [`DecodeConfig::sample_rate`] and [`DecodeConfig::sample_rate`] -- channels are interleaved
|
||||
/// (i.e., `L, R, L, R, ...`) if stereo.
|
||||
///
|
||||
/// This value will be `None` if Songbird is not configured to decode audio.
|
||||
///
|
||||
/// [`DecodeConfig::decode_channels`]: crate::driver::DecodeConfig::channels
|
||||
/// [`DecodeConfig::sample_rate`]: crate::driver::DecodeConfig::sample_rate
|
||||
pub decoded_voice: Option<Vec<i16>>,
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
use super::context_data::*;
|
||||
use crate::ConnectionInfo;
|
||||
|
||||
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
|
||||
pub struct InternalConnect {
|
||||
pub info: ConnectionInfo,
|
||||
pub ssrc: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct InternalDisconnect {
|
||||
pub kind: DisconnectKind,
|
||||
pub reason: Option<DisconnectReason>,
|
||||
pub info: ConnectionInfo,
|
||||
}
|
||||
|
||||
impl<'a> From<&'a InternalConnect> for ConnectData<'a> {
|
||||
fn from(val: &'a InternalConnect) -> Self {
|
||||
Self {
|
||||
channel_id: val.info.channel_id,
|
||||
guild_id: val.info.guild_id,
|
||||
session_id: &val.info.session_id,
|
||||
server: &val.info.endpoint,
|
||||
ssrc: val.ssrc,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> From<&'a InternalDisconnect> for DisconnectData<'a> {
|
||||
fn from(val: &'a InternalDisconnect) -> Self {
|
||||
Self {
|
||||
kind: val.kind,
|
||||
reason: val.reason,
|
||||
channel_id: val.info.channel_id,
|
||||
guild_id: val.info.guild_id,
|
||||
session_id: &val.info.session_id,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
mod receive {
|
||||
use super::*;
|
||||
use bytes::Bytes;
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct InternalRtpPacket {
|
||||
pub packet: Bytes,
|
||||
pub payload_offset: usize,
|
||||
pub payload_end_pad: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct InternalRtcpPacket {
|
||||
pub packet: Bytes,
|
||||
pub payload_offset: usize,
|
||||
pub payload_end_pad: usize,
|
||||
}
|
||||
|
||||
impl<'a> From<&'a InternalRtpPacket> for RtpData {
|
||||
fn from(val: &'a InternalRtpPacket) -> Self {
|
||||
Self {
|
||||
packet: val.packet.clone(),
|
||||
payload_offset: val.payload_offset,
|
||||
payload_end_pad: val.payload_end_pad,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> From<&'a InternalRtcpPacket> for RtcpData {
|
||||
fn from(val: &'a InternalRtcpPacket) -> Self {
|
||||
Self {
|
||||
packet: val.packet.clone(),
|
||||
payload_offset: val.payload_offset,
|
||||
payload_end_pad: val.payload_end_pad,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
pub use receive::*;
|
||||
114
app/vendor/songbird/src/events/context/mod.rs
vendored
114
app/vendor/songbird/src/events/context/mod.rs
vendored
@@ -1,114 +0,0 @@
|
||||
pub mod data;
|
||||
pub(crate) mod internal_data;
|
||||
|
||||
use super::*;
|
||||
use crate::{
|
||||
model::payload::{ClientDisconnect, Speaking},
|
||||
tracks::{TrackHandle, TrackState},
|
||||
};
|
||||
pub use data as context_data;
|
||||
use data::*;
|
||||
use internal_data::*;
|
||||
|
||||
/// Information about which tracks or data fired an event.
|
||||
///
|
||||
/// [`Track`] events may be local or global, and have no tracks
|
||||
/// if fired on the global context via [`Driver::add_global_event`].
|
||||
///
|
||||
/// [`Track`]: crate::tracks::Track
|
||||
/// [`Driver::add_global_event`]: crate::driver::Driver::add_global_event
|
||||
#[derive(Debug)]
|
||||
#[non_exhaustive]
|
||||
pub enum EventContext<'a> {
|
||||
/// Track event context, passed to events created via [`TrackHandle::add_event`],
|
||||
/// [`EventStore::add_event`], or relevant global events.
|
||||
///
|
||||
/// [`EventStore::add_event`]: EventStore::add_event
|
||||
/// [`TrackHandle::add_event`]: TrackHandle::add_event
|
||||
Track(&'a [(&'a TrackState, &'a TrackHandle)]),
|
||||
|
||||
/// Speaking state update, typically describing how another voice
|
||||
/// user is transmitting audio data. Clients must send at least one such
|
||||
/// packet to allow SSRC/UserID matching.
|
||||
SpeakingStateUpdate(Speaking),
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
/// Reordered and decoded audio packets, received every 20ms.
|
||||
VoiceTick(VoiceTick),
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
/// Opus audio packet, received from another stream.
|
||||
RtpPacket(RtpData),
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
/// Telemetry/statistics packet, received from another stream.
|
||||
RtcpPacket(RtcpData),
|
||||
|
||||
/// Fired whenever a client disconnects.
|
||||
ClientDisconnect(ClientDisconnect),
|
||||
|
||||
/// Fires when this driver successfully connects to a voice channel.
|
||||
DriverConnect(ConnectData<'a>),
|
||||
|
||||
/// Fires when this driver successfully reconnects after a network error.
|
||||
DriverReconnect(ConnectData<'a>),
|
||||
|
||||
/// Fires when this driver fails to connect to, or drops from, a voice channel.
|
||||
DriverDisconnect(DisconnectData<'a>),
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum CoreContext {
|
||||
SpeakingStateUpdate(Speaking),
|
||||
#[cfg(feature = "receive")]
|
||||
VoiceTick(VoiceTick),
|
||||
#[cfg(feature = "receive")]
|
||||
RtpPacket(InternalRtpPacket),
|
||||
#[cfg(feature = "receive")]
|
||||
RtcpPacket(InternalRtcpPacket),
|
||||
ClientDisconnect(ClientDisconnect),
|
||||
DriverConnect(InternalConnect),
|
||||
DriverReconnect(InternalConnect),
|
||||
DriverDisconnect(InternalDisconnect),
|
||||
}
|
||||
|
||||
impl<'a> CoreContext {
|
||||
pub(crate) fn to_user_context(&'a self) -> EventContext<'a> {
|
||||
match self {
|
||||
Self::SpeakingStateUpdate(evt) => EventContext::SpeakingStateUpdate(*evt),
|
||||
#[cfg(feature = "receive")]
|
||||
Self::VoiceTick(evt) => EventContext::VoiceTick(evt.clone()),
|
||||
#[cfg(feature = "receive")]
|
||||
Self::RtpPacket(evt) => EventContext::RtpPacket(RtpData::from(evt)),
|
||||
#[cfg(feature = "receive")]
|
||||
Self::RtcpPacket(evt) => EventContext::RtcpPacket(RtcpData::from(evt)),
|
||||
Self::ClientDisconnect(evt) => EventContext::ClientDisconnect(*evt),
|
||||
Self::DriverConnect(evt) => EventContext::DriverConnect(ConnectData::from(evt)),
|
||||
Self::DriverReconnect(evt) => EventContext::DriverReconnect(ConnectData::from(evt)),
|
||||
Self::DriverDisconnect(evt) =>
|
||||
EventContext::DriverDisconnect(DisconnectData::from(evt)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl EventContext<'_> {
|
||||
/// Retreive the event class for an event (i.e., when matching)
|
||||
/// an event against the registered listeners.
|
||||
#[must_use]
|
||||
pub fn to_core_event(&self) -> Option<CoreEvent> {
|
||||
match self {
|
||||
Self::SpeakingStateUpdate(_) => Some(CoreEvent::SpeakingStateUpdate),
|
||||
#[cfg(feature = "receive")]
|
||||
Self::VoiceTick(_) => Some(CoreEvent::VoiceTick),
|
||||
#[cfg(feature = "receive")]
|
||||
Self::RtpPacket(_) => Some(CoreEvent::RtpPacket),
|
||||
#[cfg(feature = "receive")]
|
||||
Self::RtcpPacket(_) => Some(CoreEvent::RtcpPacket),
|
||||
Self::ClientDisconnect(_) => Some(CoreEvent::ClientDisconnect),
|
||||
Self::DriverConnect(_) => Some(CoreEvent::DriverConnect),
|
||||
Self::DriverReconnect(_) => Some(CoreEvent::DriverReconnect),
|
||||
Self::DriverDisconnect(_) => Some(CoreEvent::DriverDisconnect),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
77
app/vendor/songbird/src/events/core.rs
vendored
77
app/vendor/songbird/src/events/core.rs
vendored
@@ -1,77 +0,0 @@
|
||||
/// Voice core events occur on receipt of
|
||||
/// voice packets and telemetry.
|
||||
///
|
||||
/// Core events persist while the `action` in [`EventData`]
|
||||
/// returns `None`.
|
||||
///
|
||||
/// ## Events from other users
|
||||
/// Songbird can observe when a user *speaks for the first time* ([`SpeakingStateUpdate`]),
|
||||
/// when a client leaves the session ([`ClientDisconnect`]).
|
||||
///
|
||||
/// When the `"receive"` feature is enabled, songbird can also handle voice packets
|
||||
#[cfg_attr(feature = "receive", doc = "([`RtpPacket`](Self::RtpPacket)),")]
|
||||
#[cfg_attr(not(feature = "receive"), doc = "(`RtpPacket`),")]
|
||||
/// decode and track speaking users
|
||||
#[cfg_attr(feature = "receive", doc = "([`VoiceTick`](Self::VoiceTick)),")]
|
||||
#[cfg_attr(not(feature = "receive"), doc = "(`VoiceTick`),")]
|
||||
/// and handle telemetry data
|
||||
#[cfg_attr(feature = "receive", doc = "([`RtcpPacket`](Self::RtcpPacket)).")]
|
||||
#[cfg_attr(not(feature = "receive"), doc = "(`RtcpPacket`).")]
|
||||
/// The format of voice packets is described by
|
||||
#[cfg_attr(
|
||||
feature = "receive",
|
||||
doc = "[`VoiceData`](super::context::data::VoiceData)."
|
||||
)]
|
||||
#[cfg_attr(not(feature = "receive"), doc = "`VoiceData`.")]
|
||||
///
|
||||
/// To detect when a user connects, you must correlate gateway (e.g., `VoiceStateUpdate`) events
|
||||
/// from the main part of your bot.
|
||||
///
|
||||
/// To obtain a user's SSRC, you must use [`SpeakingStateUpdate`] events.
|
||||
///
|
||||
/// [`EventData`]: super::EventData
|
||||
/// [`SpeakingStateUpdate`]: Self::SpeakingStateUpdate
|
||||
/// [`ClientDisconnect`]: Self::ClientDisconnect
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, enum_map::Enum)]
|
||||
#[non_exhaustive]
|
||||
pub enum CoreEvent {
|
||||
/// Speaking state update from the WS gateway, typically describing how another voice
|
||||
/// user is transmitting audio data. Clients must send at least one such
|
||||
/// packet to allow SSRC/UserID matching.
|
||||
///
|
||||
/// Fired on receipt of a speaking state update from another host.
|
||||
///
|
||||
/// Note: this will fire when a user starts speaking for the first time,
|
||||
/// or changes their capabilities.
|
||||
SpeakingStateUpdate,
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
/// Fires every 20ms, containing the scheduled voice packet and decoded audio
|
||||
/// data for each live user.
|
||||
VoiceTick,
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
/// Fires on receipt of a voice packet from another stream in the voice call.
|
||||
///
|
||||
/// As RTP packets do not map to Discord's notion of users, SSRCs must be mapped
|
||||
/// back using the user IDs seen through client connection, disconnection,
|
||||
/// or speaking state update.
|
||||
RtpPacket,
|
||||
|
||||
#[cfg(feature = "receive")]
|
||||
/// Fires on receipt of an RTCP packet, containing various call stats
|
||||
/// such as latency reports.
|
||||
RtcpPacket,
|
||||
|
||||
/// Fires whenever a user disconnects from the same stream as the bot.
|
||||
ClientDisconnect,
|
||||
|
||||
/// Fires when this driver successfully connects to a voice channel.
|
||||
DriverConnect,
|
||||
|
||||
/// Fires when this driver successfully reconnects after a network error.
|
||||
DriverReconnect,
|
||||
|
||||
/// Fires when this driver fails to connect to, or drops from, a voice channel.
|
||||
DriverDisconnect,
|
||||
}
|
||||
82
app/vendor/songbird/src/events/data.rs
vendored
82
app/vendor/songbird/src/events/data.rs
vendored
@@ -1,82 +0,0 @@
|
||||
use super::*;
|
||||
use std::cmp::Ordering;
|
||||
|
||||
/// Internal representation of an event, as handled by the audio context.
|
||||
pub struct EventData {
|
||||
pub(crate) event: Event,
|
||||
pub(crate) fire_time: Option<Duration>,
|
||||
pub(crate) action: Box<dyn EventHandler>,
|
||||
}
|
||||
|
||||
impl EventData {
|
||||
/// Create a representation of an event and its associated handler.
|
||||
///
|
||||
/// An event handler, `action`, receives an [`EventContext`] and optionally
|
||||
/// produces a new [`Event`] type for itself. Returning `None` will
|
||||
/// maintain the same event type, while removing any [`Delayed`] entries.
|
||||
/// Event handlers will be re-added with their new trigger condition,
|
||||
/// or removed if [`Cancel`]led
|
||||
///
|
||||
/// [`EventContext`]: EventContext
|
||||
/// [`Event`]: Event
|
||||
/// [`Delayed`]: Event::Delayed
|
||||
/// [`Cancel`]: Event::Cancel
|
||||
pub fn new<F: EventHandler + 'static>(event: Event, action: F) -> Self {
|
||||
Self {
|
||||
event,
|
||||
fire_time: None,
|
||||
action: Box::new(action),
|
||||
}
|
||||
}
|
||||
|
||||
/// Computes the next firing time for a timer event.
|
||||
pub fn compute_activation(&mut self, now: Duration) {
|
||||
match self.event {
|
||||
Event::Periodic(period, phase) => {
|
||||
self.fire_time = Some(now + phase.unwrap_or(period));
|
||||
},
|
||||
Event::Delayed(offset) => {
|
||||
self.fire_time = Some(now + offset);
|
||||
},
|
||||
_ => {},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for EventData {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
|
||||
write!(
|
||||
f,
|
||||
"Event {{ event: {:?}, fire_time: {:?}, action: <fn> }}",
|
||||
self.event, self.fire_time
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Events are ordered/compared based on their firing time.
|
||||
impl Ord for EventData {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
// FIXME: we don't have let chains in this edition songbird uses so when we upgrade to 2024 edition,
|
||||
// change this to a let chain that way it reads easier
|
||||
if let Some(t1) = &self.fire_time {
|
||||
if let Some(t2) = &other.fire_time {
|
||||
return t2.cmp(t1);
|
||||
}
|
||||
}
|
||||
Ordering::Equal
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialOrd for EventData {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for EventData {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.fire_time == other.fire_time
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for EventData {}
|
||||
147
app/vendor/songbird/src/events/mod.rs
vendored
147
app/vendor/songbird/src/events/mod.rs
vendored
@@ -1,147 +0,0 @@
|
||||
//! Events relating to tracks, timing, and other callers.
|
||||
//!
|
||||
//! ## Listening for events
|
||||
//! Driver events in Songbird are composed of two parts:
|
||||
//! * An [`Event`] to listen out for. These may be discrete events,
|
||||
//! or generated by timers.
|
||||
//! * An [`EventHandler`] to be called on receipt of an event. As event
|
||||
//! handlers may be shared between several events, the handler is called
|
||||
//! with an [`EventContext`] describing which event was fired.
|
||||
//!
|
||||
//! Event handlers are registered using functions such as [`Driver::add_global_event`],
|
||||
//! or [`TrackHandle::add_event`], or. Internally, these pairs are stored
|
||||
//! as [`EventData`].
|
||||
//!
|
||||
//! ## `EventHandler` lifecycle
|
||||
//! An event handler is essentially just an async function which may return
|
||||
//! another type of event to listen out for (an `Option<Event>`). For instance,
|
||||
//! [`Some(Event::Cancel)`] will remove that event listener, while `None` won't
|
||||
//! change it at all.
|
||||
//!
|
||||
//! The exception is one-off events like [`Event::Delayed`], which remove themselves
|
||||
//! after one call *unless* an [`Event`] override is returned.
|
||||
//!
|
||||
//! ## Global and local listeners
|
||||
//! *Global* event listeners are those which are placed onto the [`Driver`],
|
||||
//! while *local* event listeners are those which are placed on a
|
||||
//! [`Track`]/[`Handle`].
|
||||
//!
|
||||
//! Track or timed events, when local, return a reference to the parent track.
|
||||
//! When registered globally, they fire on a per-tick basis, returning references to
|
||||
//! all relevant tracks in that 20ms window. Global/local timed events use a global
|
||||
//! timer or a [track's playback time], respectively.
|
||||
//!
|
||||
//! [`CoreEvent`]s may only be registered globally.
|
||||
//!
|
||||
//! [`Event`]: Event
|
||||
//! [`EventHandler`]: EventHandler
|
||||
//! [`EventContext`]: EventContext
|
||||
//! [`Driver::add_global_event`]: crate::driver::Driver::add_global_event
|
||||
//! [`Driver`]: crate::driver::Driver::add_global_event
|
||||
//! [`TrackHandle::add_event`]: crate::tracks::TrackHandle::add_event
|
||||
//! [`Track`]: crate::tracks::Track::events
|
||||
//! [`Handle`]: crate::tracks::TrackHandle::add_event
|
||||
//! [`EventData`]: EventData
|
||||
//! [`Some(Event::Cancel)`]: Event::Cancel
|
||||
//! [`Event::Delayed`]: Event::Delayed
|
||||
//! [track's playback time]: crate::tracks::TrackState::play_time
|
||||
//! [`CoreEvent`]: CoreEvent
|
||||
|
||||
mod context;
|
||||
mod core;
|
||||
mod data;
|
||||
mod store;
|
||||
mod track;
|
||||
mod untimed;
|
||||
|
||||
pub use self::{
|
||||
context::{context_data, EventContext},
|
||||
core::*,
|
||||
data::*,
|
||||
store::*,
|
||||
track::*,
|
||||
untimed::*,
|
||||
};
|
||||
pub(crate) use context::{internal_data, CoreContext};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Trait to handle an event which can be fired per-track, or globally.
|
||||
///
|
||||
/// These may be feasibly reused between several event sources.
|
||||
#[async_trait]
|
||||
pub trait EventHandler: Send + Sync {
|
||||
/// Respond to one received event.
|
||||
async fn act(&self, ctx: &EventContext<'_>) -> Option<Event>;
|
||||
}
|
||||
|
||||
/// Classes of event which may occur, triggering a handler
|
||||
/// at the local (track-specific) or global level.
|
||||
///
|
||||
/// Local time-based events rely upon the current playback
|
||||
/// time of a track, and so will not fire if a track becomes paused
|
||||
/// or stops. In case this is required, global events are a better
|
||||
/// fit.
|
||||
///
|
||||
/// Event handlers themselves are described in [`EventData::new`].
|
||||
///
|
||||
/// [`EventData::new`]: EventData::new
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
#[non_exhaustive]
|
||||
pub enum Event {
|
||||
/// Periodic events rely upon two parameters: a *period*
|
||||
/// and an optional *phase*.
|
||||
///
|
||||
/// If the *phase* is `None`, then the event will first fire
|
||||
/// in one *period*. Periodic events repeat automatically
|
||||
/// so long as the `action` in [`EventData`] returns `None`.
|
||||
///
|
||||
/// [`EventData`]: EventData
|
||||
Periodic(Duration, Option<Duration>),
|
||||
/// Delayed events rely upon a *delay* parameter, and
|
||||
/// fire one *delay* after the audio context processes them.
|
||||
///
|
||||
/// Delayed events are automatically removed once fired,
|
||||
/// so long as the `action` in [`EventData`] returns `None`.
|
||||
///
|
||||
/// [`EventData`]: EventData
|
||||
Delayed(Duration),
|
||||
/// Track events correspond to certain actions or changes
|
||||
/// of state, such as a track finishing, looping, or being
|
||||
/// manually stopped.
|
||||
///
|
||||
/// Track events persist while the `action` in [`EventData`]
|
||||
/// returns `None`.
|
||||
///
|
||||
/// [`EventData`]: EventData
|
||||
Track(TrackEvent),
|
||||
/// Core events
|
||||
///
|
||||
/// Track events persist while the `action` in [`EventData`]
|
||||
/// returns `None`. Core events **must** be applied globally,
|
||||
/// as attaching them to a track is a no-op.
|
||||
///
|
||||
/// [`EventData`]: EventData
|
||||
Core(CoreEvent),
|
||||
/// Cancels the event, if it was intended to persist.
|
||||
Cancel,
|
||||
}
|
||||
|
||||
impl Event {
|
||||
pub(crate) fn is_global_only(&self) -> bool {
|
||||
matches!(self, Self::Core(_))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<TrackEvent> for Event {
|
||||
fn from(evt: TrackEvent) -> Self {
|
||||
Event::Track(evt)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<CoreEvent> for Event {
|
||||
fn from(evt: CoreEvent) -> Self {
|
||||
Event::Core(evt)
|
||||
}
|
||||
}
|
||||
263
app/vendor/songbird/src/events/store.rs
vendored
263
app/vendor/songbird/src/events/store.rs
vendored
@@ -1,263 +0,0 @@
|
||||
use super::*;
|
||||
use crate::{
|
||||
constants::*,
|
||||
tracks::{ReadyState, TrackHandle, TrackState},
|
||||
};
|
||||
use enum_map::EnumMap;
|
||||
use std::collections::{BinaryHeap, HashMap};
|
||||
use tracing::info;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
/// Storage for [`EventData`], designed to be used for both local and global contexts.
|
||||
///
|
||||
/// Timed events are stored in a binary heap for fast selection, and have custom `Eq`,
|
||||
/// `Ord`, etc. implementations to support (only) this.
|
||||
///
|
||||
/// [`EventData`]: EventData
|
||||
pub struct EventStore {
|
||||
timed: BinaryHeap<EventData>,
|
||||
untimed: Box<EnumMap<UntimedEvent, Vec<EventData>>>,
|
||||
local_only: bool,
|
||||
}
|
||||
|
||||
impl EventStore {
|
||||
/// Creates a new event store to be used globally.
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Creates a new event store to be used within a [`Track`].
|
||||
///
|
||||
/// This is usually automatically installed by the driver once
|
||||
/// a track has been registered.
|
||||
///
|
||||
/// [`Track`]: crate::tracks::Track
|
||||
#[must_use]
|
||||
pub fn new_local() -> Self {
|
||||
EventStore {
|
||||
local_only: true,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Add an event to this store.
|
||||
///
|
||||
/// Updates `evt` according to [`EventData::compute_activation`].
|
||||
///
|
||||
/// [`EventData::compute_activation`]: EventData::compute_activation
|
||||
pub fn add_event(&mut self, mut evt: EventData, now: Duration) {
|
||||
evt.compute_activation(now);
|
||||
|
||||
if self.local_only && evt.event.is_global_only() {
|
||||
return;
|
||||
}
|
||||
|
||||
match evt.event {
|
||||
Event::Core(c) => {
|
||||
self.untimed[c.into()].push(evt);
|
||||
},
|
||||
Event::Track(t) => {
|
||||
self.untimed[t.into()].push(evt);
|
||||
},
|
||||
Event::Delayed(_) | Event::Periodic(_, _) => {
|
||||
self.timed.push(evt);
|
||||
},
|
||||
_ => {
|
||||
// Event cancelled.
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Processes all events due up to and including `now`.
|
||||
pub(crate) async fn process_timed(&mut self, now: Duration, ctx: EventContext<'_>) {
|
||||
while let Some(evt) = self.timed.peek() {
|
||||
if evt
|
||||
.fire_time
|
||||
.as_ref()
|
||||
.expect("Timed event must have a fire_time.")
|
||||
> &now
|
||||
{
|
||||
break;
|
||||
}
|
||||
let mut evt = self
|
||||
.timed
|
||||
.pop()
|
||||
.expect("Can only succeed due to peek = Some(...).");
|
||||
|
||||
let old_evt_type = evt.event;
|
||||
if let Some(new_evt_type) = evt.action.act(&ctx).await {
|
||||
evt.event = new_evt_type;
|
||||
self.add_event(evt, now);
|
||||
} else if let Event::Periodic(d, _) = old_evt_type {
|
||||
evt.event = Event::Periodic(d, None);
|
||||
self.add_event(evt, now);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Processes all events due up to and including `now`.
|
||||
pub(crate) fn timed_event_ready(&self, now: Duration) -> bool {
|
||||
self.timed.peek().is_some_and(|evt| {
|
||||
evt.fire_time
|
||||
.as_ref()
|
||||
.expect("Timed event must have a fire_time.")
|
||||
<= &now
|
||||
})
|
||||
}
|
||||
|
||||
/// Processes all events attached to the given track event.
|
||||
pub(crate) async fn process_untimed(
|
||||
&mut self,
|
||||
now: Duration,
|
||||
untimed_event: UntimedEvent,
|
||||
ctx: EventContext<'_>,
|
||||
) {
|
||||
// move a Vec in and out: not too expensive, but could be better.
|
||||
// Although it's obvious that moving an event out of one vec and into
|
||||
// another necessitates that they be different event types, thus entries,
|
||||
// convincing the compiler of this is non-trivial without making them dedicated
|
||||
// fields.
|
||||
let mut events = std::mem::take(&mut self.untimed[untimed_event]);
|
||||
if !events.is_empty() {
|
||||
// TODO: Possibly use tombstones to prevent realloc/memcpys?
|
||||
// i.e., never shrink array, replace ended tracks with <DEAD>,
|
||||
// maintain a "first-track" stack and freelist alongside.
|
||||
let mut i = 0;
|
||||
while i < events.len() {
|
||||
let evt = &mut events[i];
|
||||
// Only remove/readd if the event type changes (i.e., Some AND new != old)
|
||||
if let Some(new_evt_type) = evt.action.act(&ctx).await {
|
||||
if evt.event == new_evt_type {
|
||||
let mut evt = events.remove(i);
|
||||
|
||||
evt.event = new_evt_type;
|
||||
self.add_event(evt, now);
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
} else {
|
||||
i += 1;
|
||||
};
|
||||
}
|
||||
self.untimed[untimed_event] = events;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct GlobalEvents {
|
||||
pub(crate) store: EventStore,
|
||||
pub(crate) time: Duration,
|
||||
pub(crate) awaiting_tick: HashMap<TrackEvent, Vec<usize>>,
|
||||
}
|
||||
|
||||
impl GlobalEvents {
|
||||
pub(crate) fn add_event(&mut self, evt: EventData) {
|
||||
self.store.add_event(evt, self.time);
|
||||
}
|
||||
|
||||
pub(crate) async fn fire_core_event(&mut self, evt: CoreEvent, ctx: EventContext<'_>) {
|
||||
self.store.process_untimed(self.time, evt.into(), ctx).await;
|
||||
}
|
||||
|
||||
pub(crate) fn fire_track_event(&mut self, evt: TrackEvent, index: usize) {
|
||||
let holder = self.awaiting_tick.entry(evt).or_default();
|
||||
|
||||
holder.push(index);
|
||||
}
|
||||
|
||||
pub(crate) fn remove_handlers(&mut self) {
|
||||
self.store = EventStore::new();
|
||||
}
|
||||
|
||||
pub(crate) async fn tick(
|
||||
&mut self,
|
||||
events: &mut [EventStore],
|
||||
states: &mut [TrackState],
|
||||
handles: &mut [TrackHandle],
|
||||
) {
|
||||
// Global timed events
|
||||
self.time += TIMESTEP_LENGTH;
|
||||
if self.store.timed_event_ready(self.time) {
|
||||
let global_ctx: Vec<(&TrackState, &TrackHandle)> =
|
||||
states.iter().zip(handles.iter()).collect();
|
||||
self.store
|
||||
.process_timed(self.time, EventContext::Track(&global_ctx[..]))
|
||||
.await;
|
||||
}
|
||||
|
||||
// Local timed events
|
||||
for (i, state) in states.iter_mut().enumerate() {
|
||||
if state.playing.is_playing() && state.ready == ReadyState::Playable {
|
||||
state.step_frame();
|
||||
|
||||
let event_store = events
|
||||
.get_mut(i)
|
||||
.expect("Missing store index for Tick (local timed).");
|
||||
let handle = handles
|
||||
.get_mut(i)
|
||||
.expect("Missing handle index for Tick (local timed).");
|
||||
|
||||
event_store
|
||||
.process_timed(state.play_time, EventContext::Track(&[(state, handle)]))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
for (evt, indices) in &self.awaiting_tick {
|
||||
let untimed = (*evt).into();
|
||||
|
||||
if !indices.is_empty() {
|
||||
info!("Firing {:?} for {:?}", evt, indices);
|
||||
}
|
||||
|
||||
// Local untimed track events.
|
||||
for &i in indices {
|
||||
let event_store = events
|
||||
.get_mut(i)
|
||||
.expect("Missing store index for Tick (local untimed).");
|
||||
let handle = handles
|
||||
.get_mut(i)
|
||||
.expect("Missing handle index for Tick (local untimed).");
|
||||
let state = states
|
||||
.get_mut(i)
|
||||
.expect("Missing state index for Tick (local untimed).");
|
||||
|
||||
event_store
|
||||
.process_untimed(
|
||||
state.position,
|
||||
untimed,
|
||||
EventContext::Track(&[(state, handle)]),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
// Global untimed track events.
|
||||
if !self.store.untimed[untimed].is_empty() && !indices.is_empty() {
|
||||
let global_ctx: Vec<(&TrackState, &TrackHandle)> = indices
|
||||
.iter()
|
||||
.map(|i| {
|
||||
(
|
||||
states
|
||||
.get(*i)
|
||||
.expect("Missing state index for Tick (global untimed)"),
|
||||
handles
|
||||
.get(*i)
|
||||
.expect("Missing handle index for Tick (global untimed)"),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
self.store
|
||||
.process_untimed(self.time, untimed, EventContext::Track(&global_ctx[..]))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
// Now drain vecs.
|
||||
for indices in self.awaiting_tick.values_mut() {
|
||||
indices.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
35
app/vendor/songbird/src/events/track.rs
vendored
35
app/vendor/songbird/src/events/track.rs
vendored
@@ -1,35 +0,0 @@
|
||||
// TODO: Could this be a bitset? Could accelerate lookups,
|
||||
// allow easy joint subscription & remove Vecs for related evt handling?
|
||||
|
||||
/// Track events correspond to certain actions or changes
|
||||
/// of state, such as a track finishing, looping, or being
|
||||
/// manually stopped. Voice core events occur on receipt of
|
||||
/// voice packets and telemetry.
|
||||
///
|
||||
/// Track events persist while the `action` in [`EventData`]
|
||||
/// returns `None`.
|
||||
///
|
||||
/// [`EventData`]: super::EventData
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, enum_map::Enum)]
|
||||
#[non_exhaustive]
|
||||
pub enum TrackEvent {
|
||||
/// The attached track has resumed playing.
|
||||
///
|
||||
/// This event will not fire when a track first starts,
|
||||
/// but will fire when a track changes from, e.g., paused to playing.
|
||||
/// This is most relevant for queue users: queued tracks placed into a
|
||||
/// non-empty queue are initlally paused, and are later moved to `Play`.
|
||||
Play,
|
||||
/// The attached track has been paused.
|
||||
Pause,
|
||||
/// The attached track has ended.
|
||||
End,
|
||||
/// The attached track has looped.
|
||||
Loop,
|
||||
/// The attached track is being readied or recreated.
|
||||
Preparing,
|
||||
/// The attached track has become playable.
|
||||
Playable,
|
||||
/// The attached track has encountered a runtime or initialisation error.
|
||||
Error,
|
||||
}
|
||||
29
app/vendor/songbird/src/events/untimed.rs
vendored
29
app/vendor/songbird/src/events/untimed.rs
vendored
@@ -1,29 +0,0 @@
|
||||
use super::*;
|
||||
|
||||
/// Track and voice core events.
|
||||
///
|
||||
/// Untimed events persist while the `action` in [`EventData`]
|
||||
/// returns `None`.
|
||||
///
|
||||
/// [`EventData`]: EventData
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, enum_map::Enum)]
|
||||
#[non_exhaustive]
|
||||
pub enum UntimedEvent {
|
||||
/// Untimed events belonging to a track, such as state changes, end, or loops.
|
||||
Track(TrackEvent),
|
||||
/// Untimed events belonging to the global context, such as finished tracks,
|
||||
/// client speaking updates, or RT(C)P voice and telemetry data.
|
||||
Core(CoreEvent),
|
||||
}
|
||||
|
||||
impl From<TrackEvent> for UntimedEvent {
|
||||
fn from(evt: TrackEvent) -> Self {
|
||||
UntimedEvent::Track(evt)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<CoreEvent> for UntimedEvent {
|
||||
fn from(evt: CoreEvent) -> Self {
|
||||
UntimedEvent::Core(evt)
|
||||
}
|
||||
}
|
||||
64
app/vendor/songbird/src/float_duration.rs
vendored
64
app/vendor/songbird/src/float_duration.rs
vendored
@@ -1,64 +0,0 @@
|
||||
use nonmax::NonMaxU32;
|
||||
use std::time::Duration;
|
||||
|
||||
/// A type similar to [`std::time::Duration`] except using an `f32`.
|
||||
///
|
||||
/// This limits precision, only allowing by-second precision up to ~194 days,
|
||||
/// but in turn shinks type size from `16` bytes to `4` bytes.
|
||||
///
|
||||
/// # Implementation Note
|
||||
///
|
||||
/// We actually store this as `NonMaxU32`, in order to get a niche to make `Option<FloatDuration>` the same size.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct FloatDuration(NonMaxU32);
|
||||
|
||||
impl FloatDuration {
|
||||
/// Creates a [`FloatDuration`] from a raw `f32` value.
|
||||
///
|
||||
/// # Panics
|
||||
/// This may panic if `value` is a NaN value.
|
||||
#[must_use]
|
||||
pub fn from_secs_f32(value: f32) -> Self {
|
||||
Self(NonMaxU32::new(value.to_bits()).expect("value should not be NaN"))
|
||||
}
|
||||
|
||||
/// Converts a [`FloatDuration`] to an `f32`.
|
||||
///
|
||||
/// This is cheaper than converting to a [`Duration`] and using [`Duration::as_secs_f32`] as it is more direct.
|
||||
#[must_use]
|
||||
pub fn as_secs_f32(self) -> f32 {
|
||||
f32::from_bits(self.0.get())
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for FloatDuration {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_tuple("FloatDuration")
|
||||
.field(&Duration::from(*self))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for FloatDuration {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.as_secs_f32() == other.as_secs_f32()
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialOrd for FloatDuration {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
|
||||
self.as_secs_f32().partial_cmp(&other.as_secs_f32())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Duration> for FloatDuration {
|
||||
fn from(value: Duration) -> Self {
|
||||
Self::from_secs_f32(value.as_secs_f32())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<FloatDuration> for Duration {
|
||||
fn from(value: FloatDuration) -> Self {
|
||||
Duration::from_secs_f32(value.as_secs_f32())
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user