feat(app): speak the transcription notice on joining a voice channel instead of posting it in chat
The notice (voice.joinNotice) is now a speech request in the voice channel. It is owed from each join until some of it was played or someone stopped it; when the worker cannot speak yet (auto-join while it still starts) it is spoken as soon as the worker is ready. Version 0.1.3. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2
app/Cargo.lock
generated
2
app/Cargo.lock
generated
@@ -1484,7 +1484,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.2"
|
||||
version = "0.1.3"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.2"
|
||||
version = "0.1.3"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"
|
||||
|
||||
4
app/shell/Cargo.lock
generated
4
app/shell/Cargo.lock
generated
@@ -2380,7 +2380,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.2"
|
||||
version = "0.1.3"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -2408,7 +2408,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.2"
|
||||
version = "0.1.3"
|
||||
dependencies = [
|
||||
"jarvis-app",
|
||||
"serde_json",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.2"
|
||||
version = "0.1.3"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"
|
||||
|
||||
@@ -209,6 +209,8 @@ struct Inner {
|
||||
activity: OnceLock<Arc<crate::activity::Activity>>,
|
||||
/// Speech requests whose end someone waits for (`say_to_end`), by request id.
|
||||
speech_waits: Mutex<HashMap<String, tokio::sync::oneshot::Sender<PlaybackEnd>>>,
|
||||
/// The spoken transcription notice of the current voice session.
|
||||
notice: Mutex<JoinNotice>,
|
||||
}
|
||||
|
||||
/// How long `say_to_end` waits for a reminder's speech (behind what is queued before it).
|
||||
@@ -243,6 +245,55 @@ impl UttSessions {
|
||||
}
|
||||
}
|
||||
|
||||
/// The transcription notice (`voice.joinNotice`) is spoken in the voice channel: it is owed from each
|
||||
/// join until some of it was played or someone stopped it. Speech that could not be made (the worker
|
||||
/// not connected yet) leaves it owed, and it is spoken when the worker is next ready.
|
||||
#[derive(Default)]
|
||||
pub struct JoinNotice {
|
||||
owed: bool,
|
||||
speaking: Option<String>,
|
||||
}
|
||||
|
||||
impl JoinNotice {
|
||||
pub fn joined(&mut self) {
|
||||
*self = Self { owed: true, speaking: None };
|
||||
}
|
||||
pub fn left(&mut self) {
|
||||
*self = Self::default();
|
||||
}
|
||||
/// It should be spoken now (owed and not already in the queue).
|
||||
pub fn due(&self) -> bool {
|
||||
self.owed && self.speaking.is_none()
|
||||
}
|
||||
/// It was queued as speech request `id`.
|
||||
pub fn queued(&mut self, id: String) {
|
||||
self.speaking = Some(id);
|
||||
}
|
||||
/// A speech request ended. Some(heard) when it was the notice: false means it is still owed.
|
||||
pub fn ended(&mut self, e: &PlaybackEnd) -> Option<bool> {
|
||||
if self.speaking.as_deref() != Some(e.id.as_str()) {
|
||||
return None;
|
||||
}
|
||||
self.speaking = None;
|
||||
let heard = e.played > 0 || e.cancelled;
|
||||
self.owed = !heard;
|
||||
Some(heard)
|
||||
}
|
||||
}
|
||||
|
||||
/// Queue the transcription notice when it is owed and the worker can make speech.
|
||||
fn speak_notice(inner: &Inner) {
|
||||
if inner.session.lock().unwrap().is_none() || !inner.worker.is_ready() {
|
||||
return;
|
||||
}
|
||||
with_playback(inner, |p| {
|
||||
let mut n = inner.notice.lock().unwrap();
|
||||
if n.due() {
|
||||
n.queued(p.say(&inner.settings.voice.join_notice, false));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Handle to the core; cheap to clone.
|
||||
#[derive(Clone)]
|
||||
pub struct DiscordCore {
|
||||
@@ -280,6 +331,7 @@ impl DiscordCore {
|
||||
schedules: OnceLock::new(),
|
||||
tasks: Mutex::new(HashMap::new()),
|
||||
speech_waits: Mutex::new(HashMap::new()),
|
||||
notice: Mutex::new(JoinNotice::default()),
|
||||
activity: OnceLock::new(),
|
||||
notes: OnceLock::new(),
|
||||
});
|
||||
@@ -398,7 +450,7 @@ impl DiscordCore {
|
||||
self.inner.session.lock().unwrap().as_ref().map(|s| s.channel_id)
|
||||
}
|
||||
|
||||
/// Join a voice channel of the configured server (leaving the current one) and post the notice.
|
||||
/// Join a voice channel of the configured server (leaving the current one) and speak the notice.
|
||||
pub async fn join(&self, channel_id: u64) -> Result<(), CoreError> {
|
||||
let _guard = self.inner.voice_lock.lock().await;
|
||||
let http = self.http()?.clone();
|
||||
@@ -455,11 +507,8 @@ impl DiscordCore {
|
||||
report_voice_state(&self.inner);
|
||||
debug_log!("voice", "joined voice {channel_id}");
|
||||
let _ = self.inner.events.send(CoreEvent::Joined { guild_id: gc.guild_id.get(), channel_id, session });
|
||||
let notice = CreateMessage::new().content(&self.inner.settings.voice.join_notice)
|
||||
.allowed_mentions(CreateAllowedMentions::new());
|
||||
if let Err(e) = gc.id.send_message(&http, notice).await {
|
||||
error_log!("voice", "could not post the transcription notice in {channel_id}: {e}");
|
||||
}
|
||||
self.inner.notice.lock().unwrap().joined();
|
||||
speak_notice(&self.inner);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -736,6 +785,12 @@ fn with_playback<T>(inner: &Inner, f: impl FnOnce(&mut Playback<WorkerClient, Pl
|
||||
if let Some(tx) = inner.speech_waits.lock().unwrap().remove(&end.id) {
|
||||
let _ = tx.send(end.clone());
|
||||
}
|
||||
match inner.notice.lock().unwrap().ended(end) {
|
||||
Some(true) => { debug_log!("voice", "transcription notice spoken id={} played={}", end.id, end.played); }
|
||||
Some(false) => { error_log!("voice", "could not speak the transcription notice; it is spoken when \
|
||||
the voice worker is next ready"); }
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
let _ = inner.events.send(match e {
|
||||
PlaybackEvent::Start(id) => CoreEvent::SpeakStart(id),
|
||||
@@ -753,6 +808,7 @@ async fn end_session(inner: &Arc<Inner>, s: Session, reason: &str, disconnect: b
|
||||
inner.utt_sessions.lock().unwrap().clear();
|
||||
let _ = inner.events.send(CoreEvent::Stopped);
|
||||
with_playback(inner, |p| p.stop());
|
||||
inner.notice.lock().unwrap().left();
|
||||
let _ = inner.player.send(PlayerCmd::SetCall(None));
|
||||
let (heard, lost, late, bad) = {
|
||||
let mut rx = s.rx.lock().unwrap();
|
||||
@@ -898,6 +954,7 @@ async fn worker_task(core: std::sync::Weak<Inner>, mut rx: mpsc::UnboundedReceiv
|
||||
WorkerEvent::Ready(_) => {
|
||||
report_chat_channels(&inner);
|
||||
report_voice_state(&inner);
|
||||
speak_notice(&inner);
|
||||
}
|
||||
WorkerEvent::ChatAction(a) => {
|
||||
let core = DiscordCore { inner: inner.clone() };
|
||||
@@ -1350,6 +1407,41 @@ mod tests {
|
||||
assert!(!with_playback(&core.inner, |p| p.busy()), "nothing is left in the queue");
|
||||
}
|
||||
|
||||
fn end(id: &str, played: u32, cancelled: bool, failed: bool) -> PlaybackEnd {
|
||||
PlaybackEnd { id: id.into(), played, cancelled, failed }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_notice_is_owed_from_each_join_until_it_was_heard() {
|
||||
let mut n = super::JoinNotice::default();
|
||||
assert!(!n.due(), "nothing is owed outside a voice channel");
|
||||
n.joined();
|
||||
assert!(n.due());
|
||||
n.queued("r1".into());
|
||||
assert!(!n.due(), "queued once, not twice");
|
||||
assert_eq!(n.ended(&end("r2", 1, false, false)), None, "other speech does not settle it");
|
||||
assert_eq!(n.ended(&end("r1", 2, false, false)), Some(true));
|
||||
assert!(!n.due(), "heard: not spoken again, also after a worker restart");
|
||||
n.joined();
|
||||
assert!(n.due(), "every join owes it again");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_notice_the_worker_could_not_speak_stays_owed() {
|
||||
let mut n = super::JoinNotice::default();
|
||||
n.joined();
|
||||
n.queued("r1".into());
|
||||
assert_eq!(n.ended(&end("r1", 0, false, true)), Some(false));
|
||||
assert!(n.due(), "spoken when the worker is next ready");
|
||||
// Someone stopped it: they were in the channel when it started, it is not forced on them again.
|
||||
n.queued("r2".into());
|
||||
assert_eq!(n.ended(&end("r2", 0, true, false)), Some(true));
|
||||
assert!(!n.due());
|
||||
n.joined();
|
||||
n.left();
|
||||
assert!(!n.due(), "leaving drops it");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_utterance_keeps_the_session_it_started_in() {
|
||||
let mut u = super::UttSessions::default();
|
||||
|
||||
@@ -11,8 +11,8 @@ worker side of the link is `src/voice_worker/voice_worker.spec.md`.
|
||||
## Principles
|
||||
|
||||
- **Only where the bot is.** The bot hears only the voice channel it is in and reads only the chat
|
||||
channels registered in settings with reading on. Every join posts the transcription notice in the
|
||||
voice channel's chat.
|
||||
channels registered in settings with reading on. Every join speaks the transcription notice in the
|
||||
voice channel.
|
||||
- **Chat is data.** Messages read from chat channels are handed on as data, never acted on as
|
||||
commands. Text the bot sends never pings anyone (empty allowed mentions).
|
||||
- **No text in logs.** Logs carry ids, sizes, counts and timings, never what anyone said or wrote, or
|
||||
@@ -66,7 +66,7 @@ strings (numbers would lose precision).
|
||||
| `discord.ephemeralReplies` | `true` | Command replies only the caller sees. |
|
||||
| `voice.maxSpeakers` | `10` | Most people transcribed at once, 1-10. |
|
||||
| `voice.listenToBots` | `false` | Also transcribe other bots (music bots are ignored by default). |
|
||||
| `voice.joinNotice` | "자비스가 이 음성 채널에 들어왔습니다. 이 채널의 대화는 자비스가 받아 적습니다." | Posted in the voice channel's chat on every join. Must not be empty. |
|
||||
| `voice.joinNotice` | "자비스가 이 음성 채널에 들어왔습니다. 이 채널의 대화는 자비스가 받아 적습니다." | Spoken in the voice channel on every join. Must not be empty. |
|
||||
| `voice.wakeWords` | `["자비스"]` | Sent to the worker as STT `hotwords`, and the voice wake words of call detection. |
|
||||
| `voice.stt` | `{}` | Other worker STT settings (`partials`, `end_silence_ms`, `final_beam`, `language`, ...); the worker validates them. |
|
||||
| `chatChannels` | `[]` | `{id, name, description, read (true), send (true), edit (true), react (true), delete (false), pin (false), answer ("wakeWord"), syncDiscord (true)}` per channel; `answer` is in `conversation.spec.md`, `syncDiscord` (the channel's Discord name and topic follow `name` and `description`) in `admin.spec.md`. |
|
||||
@@ -126,8 +126,12 @@ the new channel and its event handlers are replaced: leaving and joining at once
|
||||
and the late "left" update cancelled the new join (seen on real Discord).
|
||||
|
||||
- **Join** (slash command `join`, auto-join, or `join(channel_id)`): connect through songbird with DAVE
|
||||
end-to-end encryption, not deafened, then post `voice.joinNotice` in the voice channel's chat. A
|
||||
notice that cannot be posted (no permission) is logged and the bot stays.
|
||||
end-to-end encryption, not deafened, then speak `voice.joinNotice` in the voice channel (a speech
|
||||
request like any other, nothing is posted in chat). The notice is owed from the join until some of it
|
||||
was played or someone stopped it. When it cannot be spoken (the worker is not connected yet, as at
|
||||
auto-join while the worker still starts), it stays owed and is spoken as soon as the worker is ready;
|
||||
this is logged as an error and the bot stays. Leaving drops it; a worker restart later in the session
|
||||
does not repeat a notice that was heard.
|
||||
- **Hearing**: songbird only decrypts (transport and DAVE). Its own Opus decoding is not used: in
|
||||
songbird 0.6.1 it decodes the payload without trimming what DAVE decryption left at its end, which
|
||||
garbled, repeated or dropped about a third of the frames (measured with a test tone that changes
|
||||
|
||||
@@ -198,9 +198,13 @@ async fn discord_voice_core_on_real_discord() {
|
||||
ev2.lock().unwrap().push(e);
|
||||
}
|
||||
});
|
||||
core.start().await.expect("core start"); // logs in, registers the command, auto-joins, posts the notice
|
||||
core.start().await.expect("core start"); // logs in, registers the command, auto-joins, speaks the notice
|
||||
assert_eq!(core.voice_channel_id(), Some(voice));
|
||||
let jarvis = core.bot_id().unwrap();
|
||||
// The transcription notice is the first speech of the join, and it is spoken, not posted.
|
||||
until("notice spoken", 60_000, || events.lock().unwrap().iter().any(|e| matches!(e, CoreEvent::SpeakEnd(_)))).await;
|
||||
let notice = events.lock().unwrap().iter().find_map(|e| match e { CoreEvent::SpeakEnd(x) => Some(x.clone()), _ => None }).unwrap();
|
||||
assert!(!notice.failed && !notice.cancelled && notice.played > 0, "{notice:?}");
|
||||
|
||||
let call = speaker_songbird.join(GuildId::new(guild), ChannelId::new(voice)).await.expect("speaker joins");
|
||||
let ears = Ears { jarvis, ..Default::default() };
|
||||
@@ -320,7 +324,7 @@ async fn discord_voice_core_on_real_discord() {
|
||||
assert!(tail < 400, "audio went on {tail} ms after stop");
|
||||
assert!(!core.speaking());
|
||||
|
||||
// 5. Chat: live event, history with the join notice, and a send that pings nobody.
|
||||
// 5. Chat: live event, history, and a send that pings nobody.
|
||||
let marker = format!("라이브 테스트 메시지 {}", stop_at.elapsed().as_nanos());
|
||||
ChannelId::new(voice).send_message(&s_http, CreateMessage::new().content(&marker)).await.unwrap();
|
||||
until("chat event", 10_000, || events.lock().unwrap().iter().any(|e| matches!(e, CoreEvent::Chat(c) if c.content == marker))).await;
|
||||
@@ -328,7 +332,6 @@ async fn discord_voice_core_on_real_discord() {
|
||||
assert!(ev.channel_id == voice && ev.author_id == speaker_id && ev.author_is_bot && !ev.from_self);
|
||||
let history = core.read_chat(voice, 20).await.unwrap();
|
||||
assert!(history.iter().any(|m| m.content == marker && !m.from_self));
|
||||
assert!(history.iter().any(|m| m.from_self && m.content.contains("받아 적습니다")), "join notice missing");
|
||||
// An edit and a delete reach the history store.
|
||||
let mut marker_msg = ChannelId::new(voice).messages(&s_http, serenity::all::GetMessages::new().limit(5)).await.unwrap()
|
||||
.into_iter().find(|m| m.content == marker).unwrap();
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# 개인정보·법적 안전장치
|
||||
|
||||
- 모든 처리는 게임PC 안에서만 합니다. 클라우드 API를 쓰지 않습니다. 밖으로 나가는 것은 웹 검색과 날씨 조회 요청뿐입니다.
|
||||
- 봇이 음성 채널에 들어가면 "대화가 전사됩니다" 고지 메시지를 보냅니다. 문구는 설정에서 수정합니다.
|
||||
- 봇이 음성 채널에 들어가면 "대화가 전사됩니다" 고지를 음성으로 말합니다(채팅에는 올리지 않습니다). 음성 워커가 아직 준비 중이면 준비되는 대로 말합니다. 문구는 설정에서 수정합니다.
|
||||
- 봇이 참여한 음성 채널의 대화만 처리합니다. 봇이 없는 대화를 몰래 듣는 기능은 만들지 않습니다.
|
||||
- 채팅은 GUI에 등록한 채팅 채널만 읽고 저장합니다. 그 밖의 채널은 읽지 않습니다. 채팅에 올라온 글은 명령이 아니라 데이터로 다룹니다.
|
||||
- 음성 원본은 기본적으로 저장하지 않습니다. 디버그용 저장은 직접 켰을 때만 하고, 기간이 지나면 자동 삭제합니다.
|
||||
|
||||
Reference in New Issue
Block a user