feat(app): speak the transcription notice on joining a voice channel instead of posting it in chat

The notice (voice.joinNotice) is now a speech request in the voice channel.
It is owed from each join until some of it was played or someone stopped
it; when the worker cannot speak yet (auto-join while it still starts) it is
spoken as soon as the worker is ready. Version 0.1.3.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-15 23:13:31 +09:00
parent f991981776
commit d17a3bbb61
8 changed files with 119 additions and 20 deletions

2
app/Cargo.lock generated
View File

@@ -1484,7 +1484,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
[[package]]
name = "jarvis-app"
version = "0.1.2"
version = "0.1.3"
dependencies = [
"async-trait",
"base64 0.22.1",

View File

@@ -1,6 +1,6 @@
[package]
name = "jarvis-app"
version = "0.1.2"
version = "0.1.3"
edition = "2021"
rust-version = "1.85"
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"

4
app/shell/Cargo.lock generated
View File

@@ -2380,7 +2380,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
[[package]]
name = "jarvis-app"
version = "0.1.2"
version = "0.1.3"
dependencies = [
"async-trait",
"base64 0.22.1",
@@ -2408,7 +2408,7 @@ dependencies = [
[[package]]
name = "jarvis-shell"
version = "0.1.2"
version = "0.1.3"
dependencies = [
"jarvis-app",
"serde_json",

View File

@@ -1,6 +1,6 @@
[package]
name = "jarvis-shell"
version = "0.1.2"
version = "0.1.3"
edition = "2021"
rust-version = "1.85"
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"

View File

@@ -209,6 +209,8 @@ struct Inner {
activity: OnceLock<Arc<crate::activity::Activity>>,
/// Speech requests whose end someone waits for (`say_to_end`), by request id.
speech_waits: Mutex<HashMap<String, tokio::sync::oneshot::Sender<PlaybackEnd>>>,
/// The spoken transcription notice of the current voice session.
notice: Mutex<JoinNotice>,
}
/// How long `say_to_end` waits for a reminder's speech (behind what is queued before it).
@@ -243,6 +245,55 @@ impl UttSessions {
}
}
/// The transcription notice (`voice.joinNotice`) is spoken in the voice channel: it is owed from each
/// join until some of it was played or someone stopped it. Speech that could not be made (the worker
/// not connected yet) leaves it owed, and it is spoken when the worker is next ready.
#[derive(Default)]
pub struct JoinNotice {
owed: bool,
speaking: Option<String>,
}
impl JoinNotice {
pub fn joined(&mut self) {
*self = Self { owed: true, speaking: None };
}
pub fn left(&mut self) {
*self = Self::default();
}
/// It should be spoken now (owed and not already in the queue).
pub fn due(&self) -> bool {
self.owed && self.speaking.is_none()
}
/// It was queued as speech request `id`.
pub fn queued(&mut self, id: String) {
self.speaking = Some(id);
}
/// A speech request ended. Some(heard) when it was the notice: false means it is still owed.
pub fn ended(&mut self, e: &PlaybackEnd) -> Option<bool> {
if self.speaking.as_deref() != Some(e.id.as_str()) {
return None;
}
self.speaking = None;
let heard = e.played > 0 || e.cancelled;
self.owed = !heard;
Some(heard)
}
}
/// Queue the transcription notice when it is owed and the worker can make speech.
fn speak_notice(inner: &Inner) {
if inner.session.lock().unwrap().is_none() || !inner.worker.is_ready() {
return;
}
with_playback(inner, |p| {
let mut n = inner.notice.lock().unwrap();
if n.due() {
n.queued(p.say(&inner.settings.voice.join_notice, false));
}
});
}
/// Handle to the core; cheap to clone.
#[derive(Clone)]
pub struct DiscordCore {
@@ -280,6 +331,7 @@ impl DiscordCore {
schedules: OnceLock::new(),
tasks: Mutex::new(HashMap::new()),
speech_waits: Mutex::new(HashMap::new()),
notice: Mutex::new(JoinNotice::default()),
activity: OnceLock::new(),
notes: OnceLock::new(),
});
@@ -398,7 +450,7 @@ impl DiscordCore {
self.inner.session.lock().unwrap().as_ref().map(|s| s.channel_id)
}
/// Join a voice channel of the configured server (leaving the current one) and post the notice.
/// Join a voice channel of the configured server (leaving the current one) and speak the notice.
pub async fn join(&self, channel_id: u64) -> Result<(), CoreError> {
let _guard = self.inner.voice_lock.lock().await;
let http = self.http()?.clone();
@@ -455,11 +507,8 @@ impl DiscordCore {
report_voice_state(&self.inner);
debug_log!("voice", "joined voice {channel_id}");
let _ = self.inner.events.send(CoreEvent::Joined { guild_id: gc.guild_id.get(), channel_id, session });
let notice = CreateMessage::new().content(&self.inner.settings.voice.join_notice)
.allowed_mentions(CreateAllowedMentions::new());
if let Err(e) = gc.id.send_message(&http, notice).await {
error_log!("voice", "could not post the transcription notice in {channel_id}: {e}");
}
self.inner.notice.lock().unwrap().joined();
speak_notice(&self.inner);
Ok(())
}
@@ -736,6 +785,12 @@ fn with_playback<T>(inner: &Inner, f: impl FnOnce(&mut Playback<WorkerClient, Pl
if let Some(tx) = inner.speech_waits.lock().unwrap().remove(&end.id) {
let _ = tx.send(end.clone());
}
match inner.notice.lock().unwrap().ended(end) {
Some(true) => { debug_log!("voice", "transcription notice spoken id={} played={}", end.id, end.played); }
Some(false) => { error_log!("voice", "could not speak the transcription notice; it is spoken when \
the voice worker is next ready"); }
None => {}
}
}
let _ = inner.events.send(match e {
PlaybackEvent::Start(id) => CoreEvent::SpeakStart(id),
@@ -753,6 +808,7 @@ async fn end_session(inner: &Arc<Inner>, s: Session, reason: &str, disconnect: b
inner.utt_sessions.lock().unwrap().clear();
let _ = inner.events.send(CoreEvent::Stopped);
with_playback(inner, |p| p.stop());
inner.notice.lock().unwrap().left();
let _ = inner.player.send(PlayerCmd::SetCall(None));
let (heard, lost, late, bad) = {
let mut rx = s.rx.lock().unwrap();
@@ -898,6 +954,7 @@ async fn worker_task(core: std::sync::Weak<Inner>, mut rx: mpsc::UnboundedReceiv
WorkerEvent::Ready(_) => {
report_chat_channels(&inner);
report_voice_state(&inner);
speak_notice(&inner);
}
WorkerEvent::ChatAction(a) => {
let core = DiscordCore { inner: inner.clone() };
@@ -1350,6 +1407,41 @@ mod tests {
assert!(!with_playback(&core.inner, |p| p.busy()), "nothing is left in the queue");
}
fn end(id: &str, played: u32, cancelled: bool, failed: bool) -> PlaybackEnd {
PlaybackEnd { id: id.into(), played, cancelled, failed }
}
#[test]
fn the_notice_is_owed_from_each_join_until_it_was_heard() {
let mut n = super::JoinNotice::default();
assert!(!n.due(), "nothing is owed outside a voice channel");
n.joined();
assert!(n.due());
n.queued("r1".into());
assert!(!n.due(), "queued once, not twice");
assert_eq!(n.ended(&end("r2", 1, false, false)), None, "other speech does not settle it");
assert_eq!(n.ended(&end("r1", 2, false, false)), Some(true));
assert!(!n.due(), "heard: not spoken again, also after a worker restart");
n.joined();
assert!(n.due(), "every join owes it again");
}
#[test]
fn a_notice_the_worker_could_not_speak_stays_owed() {
let mut n = super::JoinNotice::default();
n.joined();
n.queued("r1".into());
assert_eq!(n.ended(&end("r1", 0, false, true)), Some(false));
assert!(n.due(), "spoken when the worker is next ready");
// Someone stopped it: they were in the channel when it started, it is not forced on them again.
n.queued("r2".into());
assert_eq!(n.ended(&end("r2", 0, true, false)), Some(true));
assert!(!n.due());
n.joined();
n.left();
assert!(!n.due(), "leaving drops it");
}
#[test]
fn an_utterance_keeps_the_session_it_started_in() {
let mut u = super::UttSessions::default();

View File

@@ -11,8 +11,8 @@ worker side of the link is `src/voice_worker/voice_worker.spec.md`.
## Principles
- **Only where the bot is.** The bot hears only the voice channel it is in and reads only the chat
channels registered in settings with reading on. Every join posts the transcription notice in the
voice channel's chat.
channels registered in settings with reading on. Every join speaks the transcription notice in the
voice channel.
- **Chat is data.** Messages read from chat channels are handed on as data, never acted on as
commands. Text the bot sends never pings anyone (empty allowed mentions).
- **No text in logs.** Logs carry ids, sizes, counts and timings, never what anyone said or wrote, or
@@ -66,7 +66,7 @@ strings (numbers would lose precision).
| `discord.ephemeralReplies` | `true` | Command replies only the caller sees. |
| `voice.maxSpeakers` | `10` | Most people transcribed at once, 1-10. |
| `voice.listenToBots` | `false` | Also transcribe other bots (music bots are ignored by default). |
| `voice.joinNotice` | "자비스가 이 음성 채널에 들어왔습니다. 이 채널의 대화는 자비스가 받아 적습니다." | Posted in the voice channel's chat on every join. Must not be empty. |
| `voice.joinNotice` | "자비스가 이 음성 채널에 들어왔습니다. 이 채널의 대화는 자비스가 받아 적습니다." | Spoken in the voice channel on every join. Must not be empty. |
| `voice.wakeWords` | `["자비스"]` | Sent to the worker as STT `hotwords`, and the voice wake words of call detection. |
| `voice.stt` | `{}` | Other worker STT settings (`partials`, `end_silence_ms`, `final_beam`, `language`, ...); the worker validates them. |
| `chatChannels` | `[]` | `{id, name, description, read (true), send (true), edit (true), react (true), delete (false), pin (false), answer ("wakeWord"), syncDiscord (true)}` per channel; `answer` is in `conversation.spec.md`, `syncDiscord` (the channel's Discord name and topic follow `name` and `description`) in `admin.spec.md`. |
@@ -126,8 +126,12 @@ the new channel and its event handlers are replaced: leaving and joining at once
and the late "left" update cancelled the new join (seen on real Discord).
- **Join** (slash command `join`, auto-join, or `join(channel_id)`): connect through songbird with DAVE
end-to-end encryption, not deafened, then post `voice.joinNotice` in the voice channel's chat. A
notice that cannot be posted (no permission) is logged and the bot stays.
end-to-end encryption, not deafened, then speak `voice.joinNotice` in the voice channel (a speech
request like any other, nothing is posted in chat). The notice is owed from the join until some of it
was played or someone stopped it. When it cannot be spoken (the worker is not connected yet, as at
auto-join while the worker still starts), it stays owed and is spoken as soon as the worker is ready;
this is logged as an error and the bot stays. Leaving drops it; a worker restart later in the session
does not repeat a notice that was heard.
- **Hearing**: songbird only decrypts (transport and DAVE). Its own Opus decoding is not used: in
songbird 0.6.1 it decodes the payload without trimming what DAVE decryption left at its end, which
garbled, repeated or dropped about a third of the frames (measured with a test tone that changes

View File

@@ -198,9 +198,13 @@ async fn discord_voice_core_on_real_discord() {
ev2.lock().unwrap().push(e);
}
});
core.start().await.expect("core start"); // logs in, registers the command, auto-joins, posts the notice
core.start().await.expect("core start"); // logs in, registers the command, auto-joins, speaks the notice
assert_eq!(core.voice_channel_id(), Some(voice));
let jarvis = core.bot_id().unwrap();
// The transcription notice is the first speech of the join, and it is spoken, not posted.
until("notice spoken", 60_000, || events.lock().unwrap().iter().any(|e| matches!(e, CoreEvent::SpeakEnd(_)))).await;
let notice = events.lock().unwrap().iter().find_map(|e| match e { CoreEvent::SpeakEnd(x) => Some(x.clone()), _ => None }).unwrap();
assert!(!notice.failed && !notice.cancelled && notice.played > 0, "{notice:?}");
let call = speaker_songbird.join(GuildId::new(guild), ChannelId::new(voice)).await.expect("speaker joins");
let ears = Ears { jarvis, ..Default::default() };
@@ -320,7 +324,7 @@ async fn discord_voice_core_on_real_discord() {
assert!(tail < 400, "audio went on {tail} ms after stop");
assert!(!core.speaking());
// 5. Chat: live event, history with the join notice, and a send that pings nobody.
// 5. Chat: live event, history, and a send that pings nobody.
let marker = format!("라이브 테스트 메시지 {}", stop_at.elapsed().as_nanos());
ChannelId::new(voice).send_message(&s_http, CreateMessage::new().content(&marker)).await.unwrap();
until("chat event", 10_000, || events.lock().unwrap().iter().any(|e| matches!(e, CoreEvent::Chat(c) if c.content == marker))).await;
@@ -328,7 +332,6 @@ async fn discord_voice_core_on_real_discord() {
assert!(ev.channel_id == voice && ev.author_id == speaker_id && ev.author_is_bot && !ev.from_self);
let history = core.read_chat(voice, 20).await.unwrap();
assert!(history.iter().any(|m| m.content == marker && !m.from_self));
assert!(history.iter().any(|m| m.from_self && m.content.contains("받아 적습니다")), "join notice missing");
// An edit and a delete reach the history store.
let mut marker_msg = ChannelId::new(voice).messages(&s_http, serenity::all::GetMessages::new().limit(5)).await.unwrap()
.into_iter().find(|m| m.content == marker).unwrap();

View File

@@ -1,7 +1,7 @@
# 개인정보·법적 안전장치
- 모든 처리는 게임PC 안에서만 합니다. 클라우드 API를 쓰지 않습니다. 밖으로 나가는 것은 웹 검색과 날씨 조회 요청뿐입니다.
- 봇이 음성 채널에 들어가면 "대화가 전사됩니다" 고지 메시지를 보냅니다. 문구는 설정에서 수정합니다.
- 봇이 음성 채널에 들어가면 "대화가 전사됩니다" 고지를 음성으로 말합니다(채팅에는 올리지 않습니다). 음성 워커가 아직 준비 중이면 준비되는 대로 말합니다. 문구는 설정에서 수정합니다.
- 봇이 참여한 음성 채널의 대화만 처리합니다. 봇이 없는 대화를 몰래 듣는 기능은 만들지 않습니다.
- 채팅은 GUI에 등록한 채팅 채널만 읽고 저장합니다. 그 밖의 채널은 읽지 않습니다. 채팅에 올라온 글은 명령이 아니라 데이터로 다룹니다.
- 음성 원본은 기본적으로 저장하지 않습니다. 디버그용 저장은 직접 켰을 때만 하고, 기간이 지나면 자동 삭제합니다.