//! The Discord voice core (`app/src/discord_core.spec.md`). //! //! One bot, one server, one voice channel at a time. Hears each speaker separately and streams their //! PCM to the voice worker, speaks text through the worker's TTS, and reads and writes the registered //! chat channels. It has no assistant logic: it reports what was heard and does what it is told. use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; use std::sync::{Arc, Mutex, OnceLock}; use async_trait::async_trait; use serenity::all::{ Cache, ChannelId, ChannelType, Client, CommandDataOptionValue, CommandInteraction, CommandOptionType, Context, CreateAllowedMentions, CreateCommand, CreateCommandOption, CreateInteractionResponse, CreateInteractionResponseMessage, CreateMessage, EditInteractionResponse, EventHandler, GatewayIntents, GetMessages, EditMessage, GuildId, Http, InstallationContext, Interaction, InteractionContext, Message, MessageId, MessageUpdateEvent, ReactionType, Ready, ShardManager, UserId, VoiceState, }; use songbird::driver::DecodeMode; use songbird::input::Input; use songbird::tracks::TrackHandle; use songbird::{Call, Config, Event, EventContext, EventHandler as VoiceEventHandler, SerenityInit, Songbird, TrackEvent}; use tokio::sync::mpsc; use crate::chat::{has_message_content_intent, split_message, ChatAccessError, ChatChannels, ChatMessage, MESSAGE_LIMIT}; use crate::playback::{ClipSink, Playback, PlaybackEnd, PlaybackEvent}; use crate::receive::{opus_payload, OpusFrames, SpeakerStream}; use crate::settings::Settings; use crate::speakers::SpeakerGate; use crate::worker_client::{ChatAction, Final, WorkerClient, WorkerEvent}; use crate::{debug_log, error_log}; /// No packet for this long and a speaker counts as stopped (Discord.js uses the same 100 ms). const SPEAKING_END: Duration = Duration::from_millis(100); #[derive(Debug, Clone, PartialEq, Eq)] pub struct Heard { pub speaker_id: u64, pub speaker_name: String, pub utt: u32, /// The voice session the utterance started in (the number its `Joined` carried); None when its /// start was not seen in a session. A transcript belongs to the session its utterance started /// in, even when it arrives after Jarvis left or moved on. pub session: Option, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct HeardFinal { pub heard: Heard, pub text: String, pub end_reason: String, pub audio_ms: u64, pub wait_ms: u64, pub stt_ms: u64, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum CoreEvent { Ready { user_id: u64, tag: String }, /// Jarvis joined a voice channel; `session` numbers each voice session from 1. Joined { guild_id: u64, channel_id: u64, session: u64 }, Left { guild_id: u64, channel_id: u64, reason: String }, SpeechStart(Heard), /// The speaker stopped sending audio (their final is still to come). SpeakingStopped(u64), Partial(Heard, String), Final(HeardFinal), SpeakStart(String), SpeakEnd(PlaybackEnd), /// All speech was stopped (`stop()`, the slash command, or the session ending). Sent before the /// `SpeakEnd`s of what it stopped. Stopped, Chat(ChatMessage), /// The brain's answer text for a `reply` (`answer()` or `chat_answer()`). ReplyText { id: String, text: String, failed: bool }, /// The brain's answer to `addressed()`. Addressed { id: String, addressed: bool }, /// The brain's answer to `rank()`: candidate ids, most important first (empty: failed). Ranked { id: String, order: Vec }, /// The voice worker connection ended: finals of utterances in progress will not come, and the /// restarted worker numbers utterances afresh. WorkerLost, /// `answer(user, ..)` queued speech `id` for a request whose speech ended at `ended_at` and whose /// final arrived at `final_at` (the dashboard times answers by it). AnswerAsked { id: String, user: u64, ended_at: Instant, final_at: Instant }, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct CoreError(pub String); impl std::fmt::Display for CoreError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(&self.0) } } impl std::error::Error for CoreError {} impl From for CoreError { fn from(e: ChatAccessError) -> Self { CoreError(e.0) } } fn cerr(what: &str) -> impl FnOnce(E) -> CoreError + '_ { move |e| CoreError(format!("{what}: {e}")) } // --- playback sink: one player task owns the songbird track of the current clip ------------------- enum PlayerCmd { Play(Vec), Stop, SetCall(Option>>), Ended(u64), } struct PlayerSink(mpsc::UnboundedSender); impl ClipSink for PlayerSink { fn play(&mut self, pcm: Vec) { let _ = self.0.send(PlayerCmd::Play(pcm)); } fn stop(&mut self) { let _ = self.0.send(PlayerCmd::Stop); } } struct TrackEnded { tx: mpsc::UnboundedSender, generation: u64, } #[async_trait] impl VoiceEventHandler for TrackEnded { async fn act(&self, _: &EventContext<'_>) -> Option { let _ = self.tx.send(PlayerCmd::Ended(self.generation)); None } } /// 48 kHz stereo PCM16 as a WAV, which songbird plays without resampling. fn wav(pcm: &[u8]) -> Vec { let mut w = Vec::with_capacity(44 + pcm.len()); let len = pcm.len() as u32; w.extend_from_slice(b"RIFF"); w.extend_from_slice(&(36 + len).to_le_bytes()); w.extend_from_slice(b"WAVEfmt "); w.extend_from_slice(&16u32.to_le_bytes()); w.extend_from_slice(&1u16.to_le_bytes()); // PCM w.extend_from_slice(&2u16.to_le_bytes()); // stereo w.extend_from_slice(&48000u32.to_le_bytes()); w.extend_from_slice(&(48000u32 * 4).to_le_bytes()); w.extend_from_slice(&4u16.to_le_bytes()); w.extend_from_slice(&16u16.to_le_bytes()); w.extend_from_slice(b"data"); w.extend_from_slice(&len.to_le_bytes()); w.extend_from_slice(pcm); w } // --- core ----------------------------------------------------------------------------------------- struct Session { guild_id: u64, channel_id: u64, rx: Arc>, /// Cleared when the session ends; stops its speaking-end timer. alive: Arc, } #[derive(Default)] struct RxState { ssrc_user: HashMap, streams: HashMap>, /// People currently sending packets, with the time of their last packet. talking: HashMap, /// Everyone whose audio reached the worker this session (they get `speaker_leave`). heard: HashSet, } struct Inner { settings: Settings, token: String, worker: WorkerClient, events: mpsc::UnboundedSender, songbird: Arc, http: OnceLock>, cache: OnceLock>, shards: OnceLock>, bot_id: OnceLock, chat: ChatChannels, chat_content: AtomicBool, gate: Mutex, playback: Mutex>, player: mpsc::UnboundedSender, session: Mutex>, /// Serialises join and leave. voice_lock: tokio::sync::Mutex<()>, names: Mutex>, /// The latest voice session's number, and the session each utterance in progress started in. voice_session: std::sync::atomic::AtomicU64, utt_sessions: Mutex, /// The reminders and scheduled tasks the brain's schedule tool adds to (admin.spec.md, "Schedules"). schedules: OnceLock>, /// Scheduled tasks waiting for the brain's answer, by reply id. tasks: Mutex>>, /// Each person's standing instructions, sent with their replies (admin.spec.md, "Personal notes"). notes: OnceLock>, /// What Jarvis did outside its answers (admin.spec.md, "Activity"). activity: OnceLock>, /// Speech requests whose end someone waits for (`say_to_end`), by request id. speech_waits: Mutex>>, /// The spoken transcription notice of the current voice session. notice: Mutex, } /// How long `say_to_end` waits for a reminder's speech (behind what is queued before it). const SPEECH_TIMEOUT: Duration = Duration::from_secs(120); /// How long a scheduled task may take, waiting for an LLM switch included (the worker gives up /// waiting for one after 180 s). const TASK_TIMEOUT: Duration = Duration::from_secs(300); /// The voice session each utterance in progress started in (`Heard::session`). #[derive(Default)] pub struct UttSessions(HashMap<(u64, u32), u64>); impl UttSessions { /// Utterance `(speaker, utt)` started; `current` is the session Jarvis is in (None: none). pub fn started(&mut self, speaker: u64, utt: u32, current: Option) { match current { Some(s) => self.0.insert((speaker, utt), s), None => self.0.remove(&(speaker, utt)), }; } pub fn session_of(&self, speaker: u64, utt: u32) -> Option { self.0.get(&(speaker, utt)).copied() } /// Its final came: nothing more of it will. pub fn finished(&mut self, speaker: u64, utt: u32) { self.0.remove(&(speaker, utt)); } /// The session ended or the worker restarted: no utterance in progress belongs to a session now. pub fn clear(&mut self) { self.0.clear(); } } /// The transcription notice (`voice.joinNotice`) is spoken in the voice channel: it is owed from each /// join until some of it was played or someone stopped it. Speech that could not be made (the worker /// not connected yet) leaves it owed, and it is spoken when the worker is next ready. #[derive(Default)] pub struct JoinNotice { owed: bool, speaking: Option, } impl JoinNotice { pub fn joined(&mut self) { *self = Self { owed: true, speaking: None }; } pub fn left(&mut self) { *self = Self::default(); } /// It should be spoken now (owed and not already in the queue). pub fn due(&self) -> bool { self.owed && self.speaking.is_none() } /// It was queued as speech request `id`. pub fn queued(&mut self, id: String) { self.speaking = Some(id); } /// A speech request ended. Some(heard) when it was the notice: false means it is still owed. pub fn ended(&mut self, e: &PlaybackEnd) -> Option { if self.speaking.as_deref() != Some(e.id.as_str()) { return None; } self.speaking = None; let heard = e.played > 0 || e.cancelled; self.owed = !heard; Some(heard) } } /// Queue the transcription notice when it is owed and the worker can make speech. fn speak_notice(inner: &Inner) { if inner.session.lock().unwrap().is_none() || !inner.worker.is_ready() { return; } with_playback(inner, |p| { let mut n = inner.notice.lock().unwrap(); if n.due() { n.queued(p.say(&inner.settings.voice.join_notice, false)); } }); } /// Handle to the core; cheap to clone. #[derive(Clone)] pub struct DiscordCore { inner: Arc, } impl DiscordCore { /// The core and its event stream. `worker_events` is the worker client's event stream. pub fn new(settings: Settings, token: String, worker: WorkerClient, worker_events: mpsc::UnboundedReceiver) -> (Self, mpsc::UnboundedReceiver) { let (events, rx) = mpsc::unbounded_channel(); let (player_tx, player_rx) = mpsc::unbounded_channel(); // songbird only decrypts; the core reorders and decodes itself (see receive.rs for why). let config = Config::default().decode_mode(DecodeMode::Decrypt); let inner = Arc::new(Inner { gate: Mutex::new(SpeakerGate::new(settings.voice.max_speakers)), chat: ChatChannels::new(&settings.chat_channels), playback: Mutex::new(Playback::new(worker.clone(), PlayerSink(player_tx.clone()))), settings, token, worker, events, songbird: Songbird::serenity_from_config(config), http: OnceLock::new(), cache: OnceLock::new(), shards: OnceLock::new(), bot_id: OnceLock::new(), chat_content: AtomicBool::new(false), player: player_tx, session: Mutex::new(None), voice_lock: tokio::sync::Mutex::new(()), names: Mutex::new(HashMap::new()), voice_session: std::sync::atomic::AtomicU64::new(0), utt_sessions: Mutex::new(UttSessions::default()), schedules: OnceLock::new(), tasks: Mutex::new(HashMap::new()), speech_waits: Mutex::new(HashMap::new()), notice: Mutex::new(JoinNotice::default()), activity: OnceLock::new(), notes: OnceLock::new(), }); tokio::spawn(player_task(Arc::downgrade(&inner), player_rx)); tokio::spawn(worker_task(Arc::downgrade(&inner), worker_events)); (Self { inner }, rx) } fn http(&self) -> Result<&Arc, CoreError> { self.inner.http.get().ok_or_else(|| CoreError("the Discord core has not been started".into())) } /// The application has the Message Content intent; without it chat channels cannot be read. pub fn chat_content(&self) -> bool { self.inner.chat_content.load(Ordering::Relaxed) } pub fn bot_id(&self) -> Option { self.inner.bot_id.get().copied() } /// Log in, register the slash command, and join the auto-join channel if one is set. pub async fn start(&self) -> Result<(), CoreError> { let inner = &self.inner; let http = Http::new(&inner.token); let app = http.get_current_application_info().await.map_err(cerr("could not reach Discord"))?; let flags = app.flags.map(|f| f.bits()).unwrap_or(0); let content = has_message_content_intent(flags); inner.chat_content.store(content, Ordering::Relaxed); let mut intents = GatewayIntents::GUILDS | GatewayIntents::GUILD_VOICE_STATES; if inner.chat.any_readable() { intents |= GatewayIntents::GUILD_MESSAGES; if content { intents |= GatewayIntents::MESSAGE_CONTENT; } else { error_log!("discord", "Message Content Intent is off for this bot; chat channels cannot be read. \ Turn it on in the Discord Developer Portal (Bot > Privileged Gateway Intents)."); } } let (ready_tx, mut ready_rx) = mpsc::unbounded_channel(); let mut client = Client::builder(&inner.token, intents) .event_handler(Handler { core: self.clone(), ready: ready_tx }) .register_songbird_with(inner.songbird.clone()) .await .map_err(cerr("could not create the Discord client"))?; let _ = inner.http.set(client.http.clone()); let _ = inner.cache.set(client.cache.clone()); let _ = inner.shards.set(client.shard_manager.clone()); let (fail_tx, mut fail_rx) = mpsc::unbounded_channel(); tokio::spawn(async move { if let Err(e) = client.start().await { let _ = fail_tx.send(e.to_string()); } }); let ready: Ready = tokio::select! { r = ready_rx.recv() => r.ok_or_else(|| CoreError("gateway closed before ready".into()))?, e = fail_rx.recv() => return Err(CoreError(format!("could not log in: {}", e.unwrap_or_default()))), }; let _ = inner.bot_id.set(ready.user.id.get()); report_chat_channels(inner); debug_log!("discord", "logged in as {} chatContent={content}", ready.user.id); let _ = inner.events.send(CoreEvent::Ready { user_id: ready.user.id.get(), tag: ready.user.tag() }); if let Err(e) = self.register_command(app.id.get()).await { // Voice and chat still work; the command can be registered on the next start. error_log!("discord", "could not register the slash command: {e}"); } if let Some(ch) = inner.settings.discord.voice_channel_id { if let Err(e) = self.join(ch).await { error_log!("voice", "auto-join {ch} failed: {e}"); } } Ok(()) } async fn register_command(&self, app_id: u64) -> Result<(), CoreError> { let Some(guild) = self.inner.settings.discord.guild_id else { error_log!("discord", "discord.guildId is not set; the slash command is not registered"); return Ok(()); }; let name = &self.inner.settings.discord.command_name; let sub = |n: &str, d: &str| CreateCommandOption::new(CommandOptionType::SubCommand, n, d); let cmd = CreateCommand::new(name) .description("자비스 음성 비서") .contexts(vec![InteractionContext::Guild]) .integration_types(vec![InstallationContext::Guild]) .add_option(sub("join", "내가 있는 음성 채널로 자비스를 부릅니다")) .add_option(sub("leave", "자비스를 음성 채널에서 내보냅니다")) .add_option(sub("stop", "자비스가 하던 말을 멈춥니다")); let http = self.http()?; http.set_application_id(app_id.into()); // Creates or updates this one command by name and leaves the app's other commands alone. GuildId::new(guild).create_command(http.as_ref(), cmd).await.map_err(cerr("create command"))?; debug_log!("discord", "slash command /{name} registered in {guild}"); Ok(()) } // --- voice ------------------------------------------------------------------------------------ /// The people in Jarvis's voice channel that it hears (ID and display name), as reported to the /// worker; empty outside a voice channel. pub fn voice_members(&self) -> Vec<(u64, String)> { let inner = &self.inner; let Some((guild, channel)) = inner.session.lock().unwrap().as_ref().map(|s| (s.guild_id, s.channel_id)) else { return vec![]; }; let users: Vec = inner.cache.get() .and_then(|c| c.guild(GuildId::new(guild)).map(|g| g.voice_states.values() .filter(|v| v.channel_id.map(|c| c.get()) == Some(channel)) .map(|v| v.user_id.get()) .collect())) .unwrap_or_default(); users.into_iter().filter(|u| should_hear(inner, *u)).map(|u| (u, name_of(inner, u))).collect() } pub fn voice_channel_id(&self) -> Option { self.inner.session.lock().unwrap().as_ref().map(|s| s.channel_id) } /// Join a voice channel of the configured server (leaving the current one) and speak the notice. pub async fn join(&self, channel_id: u64) -> Result<(), CoreError> { let _guard = self.inner.voice_lock.lock().await; let http = self.http()?.clone(); let channel = ChannelId::new(channel_id).to_channel(&http).await.map_err(cerr("could not fetch the channel"))?; let Some(gc) = channel.guild().filter(|c| matches!(c.kind, ChannelType::Voice | ChannelType::Stage)) else { return Err(CoreError(format!("channel {channel_id} is not a voice channel"))); }; if let Some(g) = self.inner.settings.discord.guild_id { if gc.guild_id.get() != g { return Err(CoreError(format!("channel {channel_id} is not in the configured server"))); } } if self.voice_channel_id() == Some(channel_id) { return Ok(()); } let current = self.inner.session.lock().unwrap().take(); if let Some(old) = current { if old.guild_id == gc.guild_id.get() { // Same server: move the existing connection. Leaving first and joining at once races // the gateway (the late "left" update cancels the new join). end_session(&self.inner, old, "moved", false).await; if let Some(call) = self.inner.songbird.get(gc.guild_id) { call.lock().await.remove_all_global_events(); } } else { end_session(&self.inner, old, "moved", true).await; } } let call = self.inner.songbird.join(gc.guild_id, gc.id).await.map_err(|e| { CoreError(format!("could not connect to voice channel {channel_id}: {e}")) }); let call = match call { Ok(c) => c, Err(e) => { let _ = self.inner.songbird.remove(gc.guild_id).await; return Err(e); } }; let rx = Arc::new(Mutex::new(RxState::default())); { let mut c = call.lock().await; let h = Receiver { core: Arc::downgrade(&self.inner), rx: rx.clone() }; c.add_global_event(CoreEvent_::SpeakingStateUpdate.into(), h.clone()); c.add_global_event(CoreEvent_::RtpPacket.into(), h.clone()); c.add_global_event(CoreEvent_::ClientDisconnect.into(), h.clone()); c.add_global_event(CoreEvent_::DriverDisconnect.into(), h); } let alive = Arc::new(AtomicBool::new(true)); tokio::spawn(speaking_end_timer(Arc::downgrade(&self.inner), rx.clone(), alive.clone())); // A new session number before any of its utterances can start. let session = self.inner.voice_session.fetch_add(1, Ordering::SeqCst) + 1; *self.inner.session.lock().unwrap() = Some(Session { guild_id: gc.guild_id.get(), channel_id, rx, alive }); let _ = self.inner.player.send(PlayerCmd::SetCall(Some(call))); report_voice_state(&self.inner); debug_log!("voice", "joined voice {channel_id}"); let _ = self.inner.events.send(CoreEvent::Joined { guild_id: gc.guild_id.get(), channel_id, session }); self.inner.notice.lock().unwrap().joined(); speak_notice(&self.inner); Ok(()) } /// Leave the voice channel. False if not in one. pub async fn leave(&self, reason: &str) -> bool { let _guard = self.inner.voice_lock.lock().await; self.leave_locked(reason).await } async fn leave_locked(&self, reason: &str) -> bool { let Some(s) = self.inner.session.lock().unwrap().take() else { return false }; end_session(&self.inner, s, reason, true).await; true } /// Speak text in the voice channel. The request id, or None when not in a voice channel. pub fn say(&self, text: &str, more: bool) -> Option { self.voice_channel_id()?; let id = with_playback(&self.inner, |p| p.say(text, more)); Some(id) } /// Speak text and wait for its end: None when not in a voice channel or when no end came within /// `SPEECH_TIMEOUT` (the speech is then cancelled). pub async fn say_to_end(&self, text: &str) -> Option { self.voice_channel_id()?; self.speak_to_end(text, SPEECH_TIMEOUT).await } async fn speak_to_end(&self, text: &str, timeout: Duration) -> Option { let (tx, rx) = tokio::sync::oneshot::channel(); // Registered before the playback's events go out, so an end at once is not missed. let id = with_playback(&self.inner, |p| { let id = p.say(text, false); self.inner.speech_waits.lock().unwrap().insert(id.clone(), tx); id }); let end = tokio::time::timeout(timeout, rx).await.ok().and_then(Result::ok); self.inner.speech_waits.lock().unwrap().remove(&id); if end.is_none() { // No end came: take it out of the queue (waiting or playing), so it never plays after // the caller went on without it (a reminder then goes to chat). with_playback(&self.inner, |p| p.cancel(&id)); } end } /// Have the brain answer `user` and speak it as one request in the queue. The request id (its /// `SpeakStart` and `SpeakEnd` follow as for `say`), or None when not in a voice channel. It ends as /// failed when the brain gives no answer. `heard`: when the request's speech ended and its final /// arrived, passed on in `AnswerAsked`. pub fn answer(&self, user: u64, text: &str, heard: (Instant, Instant)) -> Option { self.voice_channel_id()?; let id = with_playback(&self.inner, |p| p.adopt()); let _ = self.inner.events.send(CoreEvent::AnswerAsked { id: id.clone(), user, ended_at: heard.0, final_at: heard.1 }); self.inner.worker.reply_in(&id, user, text, "voice", true, None, ¬es_of(&self.inner, user)); Some(id) } /// Play the earcon in the voice channel, queued like speech. The request id, or None when not in a /// voice channel. pub fn earcon(&self) -> Option { self.voice_channel_id()?; Some(with_playback(&self.inner, |p| p.clip(crate::earcon::pcm().to_vec()))) } /// Have the brain answer a chat message written in `channel` under `id`; the text comes as /// `CoreEvent::ReplyText`. pub fn chat_answer(&self, id: &str, user: u64, channel: u64, text: &str) -> bool { self.inner.worker.reply_in(id, user, text, "chat", false, Some(channel), ¬es_of(&self.inner, user)) } /// The store the brain's schedule tool works on; without one its actions fail. pub fn set_schedules(&self, store: Arc) { let _ = self.inner.schedules.set(store); } /// The store of each person's standing instructions. pub fn set_notes(&self, notes: Arc) { let _ = self.inner.notes.set(notes); } /// The activity log chat actions are recorded in. pub fn set_activity(&self, activity: Arc) { let _ = self.inner.activity.set(activity); } /// Run a scheduled task in the brain (read-only tools, not spoken) and wait for its answer. pub async fn run_task(&self, id: &str, user: u64, text: &str) -> Option { let (tx, rx) = tokio::sync::oneshot::channel(); self.inner.tasks.lock().unwrap().insert(id.to_owned(), tx); if !self.inner.worker.reply_in(id, user, text, "task", false, None, ¬es_of(&self.inner, user)) { self.inner.tasks.lock().unwrap().remove(id); return None; } let got = tokio::time::timeout(TASK_TIMEOUT, rx).await; self.inner.tasks.lock().unwrap().remove(id); got.ok().and_then(Result::ok).filter(|t| !t.trim().is_empty()) } /// Tell the brain whether the chat answer `id` was posted. pub fn reply_delivered(&self, id: &str, ok: bool) { self.inner.worker.reply_delivered(id, ok); } /// Stop one request (an answer or a `say`); the rest of the queue goes on. pub fn cancel(&self, id: &str) { with_playback(&self.inner, |p| p.cancel(id)); } /// Ask the brain whether `text` was said to Jarvis; the answer comes as `CoreEvent::Addressed`. pub fn ask_addressed(&self, id: &str, user: u64, text: &str, people: usize, callers: usize) -> bool { self.inner.worker.addressed(id, user, text, people, callers) } /// Tell the brain a call from `user` is being said; nothing comes back. pub fn prepare(&self, user: u64, text: &str) { self.inner.worker.prepare(user, text); } /// Ask the brain to rank candidates `(id, user, text)`; the answer comes as `CoreEvent::Ranked`. pub fn ask_rank(&self, id: &str, candidates: &[(String, u64, String)]) -> bool { self.inner.worker.rank(id, candidates) } /// More text for a request started with `say(.., true)`. pub fn continue_text(&self, id: &str, text: &str, more: bool) { with_playback(&self.inner, |p| p.continue_text(id, text, more)); } /// Stop speaking at once and drop everything queued. How many requests were stopped. pub fn stop(&self) -> usize { let _ = self.inner.events.send(CoreEvent::Stopped); with_playback(&self.inner, |p| p.stop()) } pub fn speaking(&self) -> bool { self.inner.playback.lock().unwrap().busy() } // --- chat channels ---------------------------------------------------------------------------- /// The latest `limit` (1-100) messages of a registered channel, oldest first. pub async fn read_chat(&self, channel_id: u64, limit: u8) -> Result, CoreError> { self.inner.chat.check_read(channel_id)?; if !self.chat_content() { return Err(CoreError("the bot's Message Content Intent is off".into())); } let http = self.http()?; let mut msgs = ChannelId::new(channel_id) .messages(http.as_ref(), GetMessages::new().limit(limit.clamp(1, 100))) .await .map_err(cerr("could not read the channel"))?; msgs.sort_by_key(|m| m.id); Ok(msgs.iter().map(|m| to_chat(&self.inner, m)).collect()) } /// Send text to a registered channel, split at Discord's length limit. The message ids. pub async fn send_chat(&self, channel_id: u64, text: &str) -> Result, CoreError> { self.inner.chat.check_send(channel_id)?; if text.trim().is_empty() { return Err(CoreError("nothing to send".into())); } let http = self.http()?; let mut ids = vec![]; for part in split_message(text, MESSAGE_LIMIT) { // Never ping anyone from text the assistant wrote. let m = CreateMessage::new().content(part).allowed_mentions(CreateAllowedMentions::new()); let sent = ChannelId::new(channel_id).send_message(http.as_ref(), m).await.map_err(cerr("could not send"))?; ids.push(sent.id.get()); } debug_log!("chat", "chat sent to {channel_id} parts={} chars={}", ids.len(), text.chars().count()); Ok(ids) } /// Send `text` to a registered channel as a reply to `message` (the first part carries the reply; /// nobody is pinged, not even the person answered). The message ids. pub async fn send_chat_reply(&self, channel_id: u64, message: u64, text: &str) -> Result, CoreError> { self.inner.chat.check_send(channel_id)?; if text.trim().is_empty() { return Err(CoreError("nothing to send".into())); } let http = self.http()?; let mut ids = vec![]; for (i, part) in split_message(text, MESSAGE_LIMIT).into_iter().enumerate() { let mut m = CreateMessage::new().content(part).allowed_mentions(CreateAllowedMentions::new().replied_user(false)); if i == 0 { m = m.reference_message((ChannelId::new(channel_id), MessageId::new(message))); } let sent = ChannelId::new(channel_id).send_message(http.as_ref(), m).await.map_err(cerr("could not send"))?; ids.push(sent.id.get()); } debug_log!("chat", "chat answer sent to {channel_id} parts={} chars={}", ids.len(), text.chars().count()); Ok(ids) } /// Do what the brain's chat channel tool asked (chat_channels.spec.md). The message ids made or /// touched. Every action needs its channel switch; edits only the bot's own messages. pub async fn chat_action(&self, a: &ChatAction) -> Result, CoreError> { let channel: u64 = a.channel.parse().map_err(|_| CoreError(format!("bad channel id {}", a.channel)))?; self.inner.chat.check_action(channel, &a.action)?; if a.action == "send" { return self.send_chat(channel, a.text.as_deref().unwrap_or("")).await; } let id = a.message_id.as_deref().and_then(|m| m.parse::().ok()).filter(|m| *m != 0) .map(MessageId::new).ok_or_else(|| CoreError(format!("{} needs a message id", a.action)))?; let http = self.http()?; let ch = ChannelId::new(channel); match a.action.as_str() { "edit" => { let text = a.text.as_deref().unwrap_or("").trim(); if text.is_empty() { return Err(CoreError("nothing to write".into())); } if text.encode_utf16().count() > MESSAGE_LIMIT { return Err(CoreError(format!("an edit must fit one message ({MESSAGE_LIMIT} characters)"))); } let mut m = ch.message(http.as_ref(), id).await.map_err(cerr("could not fetch the message"))?; if Some(m.author.id.get()) != self.bot_id() { return Err(CoreError("Discord lets the bot edit only its own messages".into())); } m.edit(http.as_ref(), EditMessage::new().content(text).allowed_mentions(CreateAllowedMentions::new())) .await.map_err(cerr("could not edit"))?; } "react" => { let emoji = a.emoji.as_deref().unwrap_or("").trim(); let reaction = ReactionType::try_from(emoji).map_err(|_| CoreError(format!("not an emoji: {emoji}")))?; ch.create_reaction(http.as_ref(), id, reaction).await.map_err(cerr("could not react"))?; } "delete" => ch.delete_message(http.as_ref(), id).await.map_err(cerr("could not delete"))?, "pin" => ch.pin(http.as_ref(), id).await.map_err(cerr("could not pin"))?, "unpin" => ch.unpin(http.as_ref(), id).await.map_err(cerr("could not unpin"))?, other => return Err(CoreError(format!("unknown chat action {other}"))), } debug_log!("chat", "chat {} done in {channel} message={id}", a.action); Ok(vec![id.get()]) } pub async fn close(&self) { self.leave("shutdown").await; if let Some(s) = self.inner.shards.get() { s.shutdown_all().await; } } } use songbird::CoreEvent as CoreEvent_; #[async_trait] impl crate::schedule::Deliver for DiscordCore { fn hears(&self, user: u64) -> bool { self.voice_members().iter().any(|(u, _)| *u == user) } async fn say(&self, text: &str) -> bool { self.say_to_end(text).await.is_some_and(|e| !e.cancelled && !e.failed && e.played > 0) } async fn post(&self, channel: u64, text: &str) -> Result<(), String> { self.send_chat(channel, text).await.map(|_| ()).map_err(|e| e.to_string()) } async fn task(&self, id: &str, user: u64, text: &str) -> Option { self.run_task(id, user, text).await } } fn with_playback(inner: &Inner, f: impl FnOnce(&mut Playback) -> T) -> T { let (out, events) = { let mut p = inner.playback.lock().unwrap(); let out = f(&mut p); (out, p.take_events()) }; for e in events { if let PlaybackEvent::End(end) = &e { if let Some(tx) = inner.speech_waits.lock().unwrap().remove(&end.id) { let _ = tx.send(end.clone()); } match inner.notice.lock().unwrap().ended(end) { Some(true) => { debug_log!("voice", "transcription notice spoken id={} played={}", end.id, end.played); } Some(false) => { error_log!("voice", "could not speak the transcription notice; it is spoken when \ the voice worker is next ready"); } None => {} } } let _ = inner.events.send(match e { PlaybackEvent::Start(id) => CoreEvent::SpeakStart(id), PlaybackEvent::End(e) => CoreEvent::SpeakEnd(e), }); } out } /// End a session's state; with `disconnect`, also leave the voice channel (a same-server move keeps /// the connection and points it at the new channel instead). async fn end_session(inner: &Arc, s: Session, reason: &str, disconnect: bool) { s.alive.store(false, Ordering::Relaxed); // Utterances still in progress carry no session from here: their late transcripts belong to none. inner.utt_sessions.lock().unwrap().clear(); let _ = inner.events.send(CoreEvent::Stopped); with_playback(inner, |p| p.stop()); inner.notice.lock().unwrap().left(); let _ = inner.player.send(PlayerCmd::SetCall(None)); let (heard, lost, late, bad) = { let mut rx = s.rx.lock().unwrap(); let (l, t, b) = rx.streams.values().fold((0, 0, 0), |a, x| (a.0 + x.lost, a.1 + x.late, a.2 + x.bad)); (std::mem::take(&mut rx.heard), l, t, b) }; for user in heard { inner.worker.speaker_leave(&user.to_string()); } inner.gate.lock().unwrap().clear(); inner.worker.voice_leave(); if disconnect { let _ = inner.songbird.remove(GuildId::new(s.guild_id)).await; } debug_log!("voice", "left voice {} reason={reason} packets lost={lost} late={late} bad={bad}", s.channel_id); let _ = inner.events.send(CoreEvent::Left { guild_id: s.guild_id, channel_id: s.channel_id, reason: reason.into() }); } /// Tell the worker which voice channel Jarvis is in and who is there, so the history store knows /// who heard what. After a join, a move, and every (re)connection of the worker. fn report_voice_state(inner: &Inner) { let Some((guild, channel)) = inner.session.lock().unwrap().as_ref().map(|s| (s.guild_id, s.channel_id)) else { return; }; inner.worker.voice_join(channel); let users: Vec = inner.cache.get() .and_then(|c| c.guild(GuildId::new(guild)).map(|g| g.voice_states.values() .filter(|v| v.channel_id.map(|c| c.get()) == Some(channel)) .map(|v| v.user_id.get()) .collect())) .unwrap_or_default(); let mut n = 0; for user in users.into_iter().filter(|u| should_hear(inner, *u)) { inner.worker.member_join(user, &name_of(inner, user)); n += 1; } debug_log!("voice", "reported voice {channel} with {n} members to the worker"); } /// The registered chat channels and their switches, for the brain's chat tool. fn report_chat_channels(inner: &Inner) { inner.worker.chat_channels(inner.bot_id.get().copied(), &inner.settings.chat_channels); } /// The person's standing instructions for a reply; none with `features.notes` off. fn notes_of(inner: &Inner, user: u64) -> Vec { match inner.notes.get() { Some(n) if inner.settings.features.notes => n.texts_of(user), _ => vec![], } } fn should_hear(inner: &Inner, user: u64) -> bool { if Some(user) == inner.bot_id.get().copied() { return false; } if inner.settings.voice.listen_to_bots { return true; } // Unknown users are people: the voice state that put a bot in the channel would have cached it. !inner.cache.get().and_then(|c| c.user(UserId::new(user)).map(|u| u.bot)).unwrap_or(false) } fn name_of(inner: &Inner, user: u64) -> String { if let Some(n) = inner.names.lock().unwrap().get(&user) { return n.clone(); } let guild = inner.session.lock().unwrap().as_ref().map(|s| s.guild_id).or(inner.settings.discord.guild_id); let name = inner.cache.get().and_then(|c| { guild .and_then(|g| c.guild(GuildId::new(g))?.members.get(&UserId::new(user)).map(|m| m.display_name().to_owned())) .or_else(|| c.user(UserId::new(user)).map(|u| u.display_name().to_owned())) }); match name { Some(n) => { inner.names.lock().unwrap().insert(user, n.clone()); n } None => user.to_string(), } } fn heard(inner: &Inner, speaker: &str, utt: u32) -> Option { let id: u64 = speaker.parse().ok()?; let session = inner.utt_sessions.lock().unwrap().session_of(id, utt); Some(Heard { speaker_id: id, speaker_name: name_of(inner, id), utt, session }) } /// An utterance started: it belongs to the session Jarvis is in now (none outside one). fn utterance_started(inner: &Inner, speaker: &str, utt: u32) { let Ok(id) = speaker.parse::() else { return }; let current = inner.session.lock().unwrap().is_some().then(|| inner.voice_session.load(Ordering::SeqCst)); inner.utt_sessions.lock().unwrap().started(id, utt, current); } fn to_chat(inner: &Inner, m: &Message) -> ChatMessage { let name = m.member.as_ref().and_then(|p| p.nick.clone()) .unwrap_or_else(|| m.author.display_name().to_owned()); ChatMessage { channel_id: m.channel_id.get(), message_id: m.id.get(), author_id: m.author.id.get(), author_name: name, author_is_bot: m.author.bot, from_self: Some(m.author.id.get()) == inner.bot_id.get().copied(), content: m.content.clone(), created_at: m.timestamp.to_rfc3339().unwrap_or_default(), } } async fn worker_task(core: std::sync::Weak, mut rx: mpsc::UnboundedReceiver) { while let Some(ev) = rx.recv().await { let Some(inner) = core.upgrade() else { return }; match ev { WorkerEvent::SpeechStart(m) => { utterance_started(&inner, &m.speaker, m.utt); if let Some(h) = heard(&inner, &m.speaker, m.utt) { let _ = inner.events.send(CoreEvent::SpeechStart(h)); } } WorkerEvent::Partial(m) => { if let Some(h) = heard(&inner, &m.speaker, m.utt) { let _ = inner.events.send(CoreEvent::Partial(h, m.text)); } } WorkerEvent::Final(m) => on_final(&inner, m), WorkerEvent::TtsAudio { id, seq, pcm } => with_playback(&inner, |p| p.audio(&id, seq, pcm)), WorkerEvent::TtsDone(d) => with_playback(&inner, |p| p.done(&d)), WorkerEvent::ReplyText { id, text, failed } => { debug_log!("voice", "reply {id} chars={} failed={failed}", text.chars().count()); if let Some(tx) = inner.tasks.lock().unwrap().remove(&id) { let _ = tx.send(if failed { String::new() } else { text }); continue; } let _ = inner.events.send(CoreEvent::ReplyText { id, text, failed }); } WorkerEvent::Addressed { id, addressed } => { let _ = inner.events.send(CoreEvent::Addressed { id, addressed }); } WorkerEvent::Ranked { id, order } => { let _ = inner.events.send(CoreEvent::Ranked { id, order }); } WorkerEvent::Ready(_) => { report_chat_channels(&inner); report_voice_state(&inner); speak_notice(&inner); } WorkerEvent::ChatAction(a) => { let core = DiscordCore { inner: inner.clone() }; tokio::spawn(async move { let result = core.chat_action(&a).await.map_err(|e| e.to_string()); if let Err(e) = &result { debug_log!("chat", "chat {} {} failed: {e}", a.action, a.id); } if let Some(activity) = core.inner.activity.get() { // Ids and known words only: the model may put any text in these fields, // and errors may quote it. use crate::activity::{chat_kind, chat_target, id_or_mark, CHAT_FAILED}; let target = chat_target(&a.channel, a.message_id.as_deref()); let by = a.user.as_ref().map(|u| match u.parse() { Ok(id) if id_or_mark(u) == *u => format!("{u} {}", name_of(&core.inner, id)), _ => id_or_mark(u), }); activity.record(&chat_kind(&a.action), &target, by, result.is_ok(), result.is_err().then(|| CHAT_FAILED.to_owned())); } core.inner.worker.chat_action_result(&a.id, result); }); } WorkerEvent::NoteAction(a) => { let result = match inner.notes.get() { Some(notes) => { let name = a.user.parse().map(|u| name_of(&inner, u)).unwrap_or_else(|_| a.user.clone()); crate::notes::act(notes, &a, &name, chrono::Utc::now().timestamp()) } None => Err("notes are not available".into()), }; if let Err(e) = &result { debug_log!("conversation", "note {} {} refused: {e}", a.action, a.id); } inner.worker.note_action_result(&a.id, result); } WorkerEvent::ScreenAction(a) => { let inner = inner.clone(); tokio::spawn(async move { let started = std::time::Instant::now(); let settings = inner.settings.clone(); let req = a.clone(); let result = tokio::task::spawn_blocking(move || match crate::screen::system_screen() { Some(screen) => crate::screen::act(&settings, screen.as_ref(), &req), None => Err("이 PC에서는 화면을 볼 수 없습니다".into()), }).await.unwrap_or_else(|_| Err("화면을 캡처하지 못했습니다".into())); // The kind of target, size and time only: never a title or a picture. let target = crate::screen::target_of(&a).map(|(t, _)| crate::screen::activity_target(&t)) .unwrap_or_else(|_| "?".into()); match &result { Ok(v) => debug_log!("conversation", "screen {} {target} {}x{} parts={} ms={}", a.action, v["item"]["width"], v["item"]["height"], v["item"]["parts"], started.elapsed().as_millis()), Err(e) => debug_log!("conversation", "screen {} {target} refused: {e}", a.action), } if a.action == "capture" { if let Some(activity) = inner.activity.get() { let by = a.user.parse().ok().map(|u| format!("{u} {}", name_of(&inner, u))); activity.record("screen.view", &target, by, result.is_ok(), result.as_ref().err().cloned()); } } inner.worker.screen_action_result(&a.id, result); }); } WorkerEvent::ScheduleAction(a) => { let result = match inner.schedules.get() { Some(store) => { let name = a.user.parse().map(|u| name_of(&inner, u)).unwrap_or_else(|_| a.user.clone()); crate::schedule::act(store, &a, &name, chrono::Utc::now().timestamp()) } None => Err("schedules are not available".into()), }; if let Err(e) = &result { debug_log!("schedule", "schedule {} {} refused: {e}", a.action, a.id); } inner.worker.schedule_action_result(&a.id, result); } WorkerEvent::Disconnected => { // The restarted worker numbers utterances afresh: the old ones' sessions are void. inner.utt_sessions.lock().unwrap().clear(); // Its tasks will not be answered. inner.tasks.lock().unwrap().clear(); let _ = inner.events.send(CoreEvent::WorkerLost); } WorkerEvent::Error(_) => {} } } } fn on_final(inner: &Inner, m: Final) { debug_log!("voice", "final speaker={} utt={} reason={} chars={} audio_ms={} wait_ms={}", m.speaker, m.utt, m.end_reason, m.text.chars().count(), m.audio_ms, m.wait_ms); let h = heard(inner, &m.speaker, m.utt); if let Ok(id) = m.speaker.parse::() { inner.utt_sessions.lock().unwrap().finished(id, m.utt); } if let Some(h) = h { let _ = inner.events.send(CoreEvent::Final(HeardFinal { heard: h, text: m.text, end_reason: m.end_reason, audio_ms: m.audio_ms, wait_ms: m.wait_ms, stt_ms: m.stt_ms })); } } async fn player_task(core: std::sync::Weak, mut rx: mpsc::UnboundedReceiver) { let mut call: Option>> = None; let mut current: Option<(TrackHandle, u64)> = None; let mut generation = 0u64; while let Some(cmd) = rx.recv().await { let Some(inner) = core.upgrade() else { return }; match cmd { PlayerCmd::SetCall(c) => { if let Some((t, _)) = current.take() { let _ = t.stop(); } call = c; } PlayerCmd::Play(pcm) => { generation += 1; let Some(c) = &call else { // Nowhere to play: count it as played so the queue moves on. with_playback(&inner, |p| p.clip_ended()); continue; }; let track = c.lock().await.play_only_input(Input::from(wav(&pcm))); let ended = TrackEnded { tx: inner.player.clone(), generation }; let _ = track.add_event(Event::Track(TrackEvent::End), ended); let ended = TrackEnded { tx: inner.player.clone(), generation }; let _ = track.add_event(Event::Track(TrackEvent::Error), ended); current = Some((track, generation)); } PlayerCmd::Stop => { if let Some((t, _)) = current.take() { let _ = t.stop(); } } PlayerCmd::Ended(g) => { // Only the end of the clip we started counts; a stop() already moved on. if current.as_ref().is_some_and(|(_, cg)| *cg == g) { current = None; with_playback(&inner, |p| p.clip_ended()); } } } } } // --- receive -------------------------------------------------------------------------------------- #[derive(Clone)] struct Receiver { core: std::sync::Weak, /// This session's receive state; also tells this session apart from a later one. rx: Arc>, } impl Receiver { fn packet(&self, inner: &Inner, rtp: &songbird::events::context_data::RtpData) { let Some((seq, ssrc, opus)) = opus_payload(&rtp.packet, rtp.payload_offset, rtp.payload_end_pad) else { return }; let mut rx = self.rx.lock().unwrap(); let Some(&user) = rx.ssrc_user.get(&ssrc) else { return }; if !should_hear(inner, user) { return; } let mut gate = inner.gate.lock().unwrap(); if rx.talking.insert(user, Instant::now()).is_none() && !gate.start(user) { debug_log!("voice", "speaker {user} not heard: {} already speaking", gate.count()); } // Decode even when not heard, so the decoder state stays continuous. let frames = rx.streams.entry(ssrc).or_insert_with(|| SpeakerStream::new(OpusFrames::new())).push(seq, opus.to_vec()); if gate.is_active(user) && !frames.is_empty() { rx.heard.insert(user); let id = user.to_string(); for f in frames { inner.worker.send_audio(&id, &f); } } } fn left(&self, inner: &Inner, user: u64) { let mut rx = self.rx.lock().unwrap(); rx.talking.remove(&user); let ssrcs: Vec = rx.ssrc_user.iter().filter(|(_, u)| **u == user).map(|(s, _)| *s).collect(); for ssrc in ssrcs { rx.ssrc_user.remove(&ssrc); rx.streams.remove(&ssrc); } let was_heard = inner.gate.lock().unwrap().end(user); if rx.heard.remove(&user) || was_heard { inner.worker.speaker_leave(&user.to_string()); } } } #[async_trait] impl VoiceEventHandler for Receiver { async fn act(&self, ctx: &EventContext<'_>) -> Option { let inner = self.core.upgrade()?; match ctx { EventContext::SpeakingStateUpdate(s) => { if let Some(u) = s.user_id { self.rx.lock().unwrap().ssrc_user.insert(s.ssrc, u.0); } } EventContext::RtpPacket(p) => self.packet(&inner, p), EventContext::ClientDisconnect(d) => self.left(&inner, d.user_id.0), // No reason: we asked for it (leave or move), and that path already ended the session. EventContext::DriverDisconnect(d) if d.reason.is_some() => { // The driver gave up reconnecting (kicked, channel deleted, long network loss). let core = DiscordCore { inner: inner.clone() }; let rx = self.rx.clone(); tokio::spawn(async move { let _g = core.inner.voice_lock.lock().await; let s = { let mut s = core.inner.session.lock().unwrap(); // Only the session this handler belongs to: a late event from an old // connection must not end a newer one. if s.as_ref().is_some_and(|s| Arc::ptr_eq(&s.rx, &rx)) { s.take() } else { None } }; if let Some(s) = s { end_session(&core.inner, s, "disconnected", true).await; } }); } _ => {} } None } } /// Every 20 ms: anyone without a packet for SPEAKING_END has stopped. Play out what their reorder /// window still holds, then tell the worker. async fn speaking_end_timer(core: std::sync::Weak, rx: Arc>, alive: Arc) { let mut every = tokio::time::interval(Duration::from_millis(20)); while alive.load(Ordering::Relaxed) { every.tick().await; let Some(inner) = core.upgrade() else { return }; let mut rx = rx.lock().unwrap(); let now = Instant::now(); let ended: Vec = rx.talking.iter().filter(|(_, t)| now.duration_since(**t) >= SPEAKING_END).map(|(u, _)| *u).collect(); for user in ended { rx.talking.remove(&user); let ssrcs: Vec = rx.ssrc_user.iter().filter(|(_, u)| **u == user).map(|(s, _)| *s).collect(); let mut gate = inner.gate.lock().unwrap(); let active = gate.is_active(user); let id = user.to_string(); for ssrc in ssrcs { if let Some(st) = rx.streams.get_mut(&ssrc) { for f in st.flush() { if active { inner.worker.send_audio(&id, &f); } } } } if gate.end(user) { inner.worker.speaking_end(&id); let _ = inner.events.send(CoreEvent::SpeakingStopped(user)); } } } } // --- gateway events ------------------------------------------------------------------------------- struct Handler { core: DiscordCore, ready: mpsc::UnboundedSender, } #[async_trait] impl EventHandler for Handler { async fn ready(&self, _: Context, ready: Ready) { let _ = self.ready.send(ready); } async fn voice_state_update(&self, _: Context, old: Option, new: VoiceState) { let inner = &self.core.inner; let user = new.user_id.get(); if let Some(m) = &new.member { inner.names.lock().unwrap().insert(user, m.display_name().to_owned()); } let (session_channel, rx) = { let s = inner.session.lock().unwrap(); match s.as_ref() { Some(s) if new.guild_id.map(|g| g.get()) == Some(s.guild_id) => (s.channel_id, s.rx.clone()), _ => return, } }; let new_channel = new.channel_id.map(|c| c.get()); if Some(user) == inner.bot_id.get().copied() { // Moved by someone else: follow the new channel id. Disconnects end via DriverDisconnect. if let Some(c) = new_channel.filter(|c| *c != session_channel) { if let Some(s) = inner.session.lock().unwrap().as_mut() { s.channel_id = c; } report_voice_state(inner); } return; } let was_here = old.and_then(|o| o.channel_id).map(|c| c.get()) == Some(session_channel); let is_here = new_channel == Some(session_channel); if was_here && !is_here { Receiver { core: Arc::downgrade(inner), rx }.left(inner, user); inner.worker.member_leave(user); } else if is_here && !was_here && should_hear(inner, user) { // Also fires for a mute or deafen when the old state was not cached; the worker keeps // one open presence per person, so that is harmless. inner.worker.member_join(user, &name_of(inner, user)); } } async fn message(&self, _: Context, m: Message) { let inner = &self.core.inner; let ch = m.channel_id.get(); if !self.core.chat_content() || !inner.chat.can_read(ch) { return; } let c = to_chat(inner, &m); // The history keeps Jarvis's own messages too; a reply to someone's message is an answer to them. let to = c.from_self.then(|| m.referenced_message.as_ref().map(|r| r.author.id.get())).flatten(); inner.worker.chat_message(c.channel_id, c.message_id, c.author_id, &c.author_name, &c.content, m.timestamp.unix_timestamp() as f64, to); if c.from_self { return; } debug_log!("chat", "chat message in {ch} from {} chars={}", m.author.id, m.content.chars().count()); let _ = inner.events.send(CoreEvent::Chat(c)); } async fn message_update(&self, _: Context, _: Option, _: Option, e: MessageUpdateEvent) { let ch = e.channel_id.get(); if !self.core.chat_content() || !self.core.inner.chat.can_read(ch) { return; } if let Some(text) = &e.content { debug_log!("chat", "chat message {} edited in {ch}", e.id); self.core.inner.worker.chat_edit(ch, e.id.get(), text); } } async fn message_delete(&self, _: Context, channel: ChannelId, id: MessageId, _: Option) { if self.core.inner.chat.can_read(channel.get()) { debug_log!("chat", "chat message {id} deleted in {channel}"); self.core.inner.worker.chat_delete(channel.get(), id.get()); } } async fn message_delete_bulk(&self, _: Context, channel: ChannelId, ids: Vec, _: Option) { if self.core.inner.chat.can_read(channel.get()) { debug_log!("chat", "{} chat messages deleted in {channel}", ids.len()); for id in ids { self.core.inner.worker.chat_delete(channel.get(), id.get()); } } } async fn interaction_create(&self, _: Context, interaction: Interaction) { if let Interaction::Command(cmd) = interaction { self.on_command(cmd).await; } } } impl Handler { async fn on_command(&self, cmd: CommandInteraction) { let inner = &self.core.inner; if cmd.data.name != inner.settings.discord.command_name { return; } let Some(guild) = cmd.guild_id else { return }; if inner.settings.discord.guild_id.is_some_and(|g| g != guild.get()) { return; } let Ok(http) = self.core.http().cloned() else { return }; let eph = inner.settings.discord.ephemeral_replies; let sub = cmd.data.options.first().filter(|o| matches!(o.value, CommandDataOptionValue::SubCommand(_))) .map(|o| o.name.clone()).unwrap_or_default(); debug_log!("discord", "command {sub} from {}", cmd.user.id); let reply = |content: &str| CreateInteractionResponse::Message( CreateInteractionResponseMessage::new().content(content).ephemeral(eph)); let result: Result<(), CoreError> = async { match sub.as_str() { "join" => { let channel = inner.cache.get().and_then(|c| { c.guild(guild).and_then(|g| g.voice_states.get(&cmd.user.id).and_then(|v| v.channel_id)) }); let Some(channel) = channel else { cmd.create_response(&http, reply("먼저 음성 채널에 들어간 뒤 다시 불러 주세요.")).await .map_err(cerr("reply"))?; return Ok(()); }; cmd.create_response(&http, CreateInteractionResponse::Defer( CreateInteractionResponseMessage::new().ephemeral(eph))).await.map_err(cerr("defer"))?; self.core.join(channel.get()).await?; cmd.edit_response(&http, EditInteractionResponse::new().content("음성 채널에 들어왔습니다.")).await .map_err(cerr("reply"))?; } "leave" => { let left = self.core.leave("command").await; cmd.create_response(&http, reply(if left { "음성 채널에서 나갔습니다." } else { "음성 채널에 있지 않습니다." })) .await.map_err(cerr("reply"))?; } "stop" => { let stopped = self.core.stop(); cmd.create_response(&http, reply(if stopped > 0 { "멈췄습니다." } else { "지금은 말하고 있지 않습니다." })) .await.map_err(cerr("reply"))?; } _ => {} } Ok(()) } .await; if let Err(e) = result { error_log!("discord", "command {sub} failed: {e}"); let content = "처리하지 못했습니다. 잠시 뒤 다시 시도해 주세요."; if cmd.edit_response(&http, EditInteractionResponse::new().content(content)).await.is_err() { let _ = cmd.create_response(&http, reply(content)).await; } } } } #[cfg(test)] mod tests { /// A reminder's speech that waits too long behind other speech is taken out of the queue, so it /// never plays after the reminder already went to chat. #[tokio::test] async fn speech_that_times_out_is_cancelled_and_never_plays_later() { let (worker, worker_events) = WorkerClient::new(crate::worker_client::WorkerClientOptions::new("ws://127.0.0.1:9", "t")); let (core, mut events) = DiscordCore::new(Settings::default(), String::new(), worker, worker_events); // An answer at the head of the queue that gets no audio yet holds everything behind it. let blocker = with_playback(&core.inner, |p| p.adopt()); let end = core.speak_to_end("철수님, 회의", Duration::from_millis(200)).await; assert!(end.as_ref().is_none_or(|e| e.cancelled || e.failed), "{end:?}"); with_playback(&core.inner, |p| p.fail(&blocker)); tokio::time::sleep(Duration::from_millis(100)).await; let mut started = vec![]; let mut ends = vec![]; while let Ok(ev) = events.try_recv() { match ev { CoreEvent::SpeakStart(id) => started.push(id), CoreEvent::SpeakEnd(e) => ends.push(e), _ => {} } } let reminder: Vec<_> = ends.iter().filter(|e| e.id != blocker).collect(); assert_eq!(reminder.len(), 1, "{ends:?}"); // Taken out at the timeout, not when the queue reached it. assert_eq!(ends[0].id, reminder[0].id, "{ends:?}"); assert!(reminder[0].cancelled && reminder[0].played == 0, "{reminder:?}"); assert!(!started.contains(&reminder[0].id), "the reminder started after it timed out: {started:?}"); assert!(!with_playback(&core.inner, |p| p.busy()), "nothing is left in the queue"); } fn end(id: &str, played: u32, cancelled: bool, failed: bool) -> PlaybackEnd { PlaybackEnd { id: id.into(), played, cancelled, failed } } #[test] fn the_notice_is_owed_from_each_join_until_it_was_heard() { let mut n = super::JoinNotice::default(); assert!(!n.due(), "nothing is owed outside a voice channel"); n.joined(); assert!(n.due()); n.queued("r1".into()); assert!(!n.due(), "queued once, not twice"); assert_eq!(n.ended(&end("r2", 1, false, false)), None, "other speech does not settle it"); assert_eq!(n.ended(&end("r1", 2, false, false)), Some(true)); assert!(!n.due(), "heard: not spoken again, also after a worker restart"); n.joined(); assert!(n.due(), "every join owes it again"); } #[test] fn a_notice_the_worker_could_not_speak_stays_owed() { let mut n = super::JoinNotice::default(); n.joined(); n.queued("r1".into()); assert_eq!(n.ended(&end("r1", 0, false, true)), Some(false)); assert!(n.due(), "spoken when the worker is next ready"); // Someone stopped it: they were in the channel when it started, it is not forced on them again. n.queued("r2".into()); assert_eq!(n.ended(&end("r2", 0, true, false)), Some(true)); assert!(!n.due()); n.joined(); n.left(); assert!(!n.due(), "leaving drops it"); } #[test] fn an_utterance_keeps_the_session_it_started_in() { let mut u = super::UttSessions::default(); u.started(11, 7, Some(1)); // in channel A u.started(22, 1, None); // started outside a session // Jarvis moves to B (session 2): A's utterance still belongs to session 1. u.started(11, 8, Some(2)); assert_eq!((u.session_of(11, 7), u.session_of(11, 8), u.session_of(22, 1)), (Some(1), Some(2), None)); u.finished(11, 7); assert_eq!(u.session_of(11, 7), None); u.clear(); assert_eq!(u.session_of(11, 8), None); } use super::*; #[tokio::test] async fn stop_reports_itself_before_the_speech_it_ends() { let (worker, worker_events) = WorkerClient::new(crate::worker_client::WorkerClientOptions::new("ws://127.0.0.1:1", "t")); let (core, mut events) = DiscordCore::new(Settings::default(), "token".into(), worker, worker_events); let id = with_playback(&core.inner, |p| p.adopt()); assert_eq!(core.stop(), 1); assert_eq!(events.recv().await, Some(CoreEvent::Stopped)); assert!(matches!(events.recv().await, Some(CoreEvent::SpeakEnd(e)) if e.id == id && e.cancelled)); } #[test] fn wav_header_describes_48k_stereo_pcm16() { let w = wav(&[1, 2, 3, 4]); assert_eq!(&w[0..4], b"RIFF"); assert_eq!(u32::from_le_bytes(w[4..8].try_into().unwrap()), 40); assert_eq!(u16::from_le_bytes(w[22..24].try_into().unwrap()), 2); assert_eq!(u32::from_le_bytes(w[24..28].try_into().unwrap()), 48000); assert_eq!(u32::from_le_bytes(w[40..44].try_into().unwrap()), 4); assert_eq!(&w[44..], &[1, 2, 3, 4]); } }