Compare commits
2 Commits
app-v0.1.2
...
app-v0.1.4
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dbe5c617bc | ||
|
|
d17a3bbb61 |
2
app/Cargo.lock
generated
2
app/Cargo.lock
generated
@@ -1484,7 +1484,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.2"
|
||||
version = "0.1.4"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.2"
|
||||
version = "0.1.4"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"
|
||||
|
||||
4
app/shell/Cargo.lock
generated
4
app/shell/Cargo.lock
generated
@@ -2380,7 +2380,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.2"
|
||||
version = "0.1.4"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -2408,7 +2408,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.2"
|
||||
version = "0.1.4"
|
||||
dependencies = [
|
||||
"jarvis-app",
|
||||
"serde_json",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.2"
|
||||
version = "0.1.4"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"
|
||||
|
||||
@@ -660,7 +660,7 @@ back as Korean messages for the screen. The Tauri shell passes the screens' `inv
|
||||
| `token_check` `{token}` | the bot's tag and IDs, whether the Message Content intent is on, the invite link (REST only, nothing stored) |
|
||||
| `token_save` `{token}` / `token_status` | store it (masked from then on in logs) / `set` and the masked form, never the token |
|
||||
| `discord_guilds` / `discord_channels` `{guildId}` | the bot's servers / a server's voice and text channels, for the pickers |
|
||||
| `settings_fields` / `settings_view` / `settings_apply` `{changes}` | the settings editor above; `restartNeeded` when the services run; `sync` with the chat channels sent to Discord (below) |
|
||||
| `settings_fields` / `settings_view` / `settings_apply` `{changes}` | the settings editor above; `restartNeeded` when the services run and something other than `voice.tts.speed` changed (the speed reaches the running worker at once, `discord_core.spec.md`); `sync` with the chat channels sent to Discord (below) |
|
||||
| `service_start` / `service_stop` / `service_restart` / `dashboard` | the controller and its snapshot; start and restart run in the background, stop also cancels a start |
|
||||
| `discord_channel_info` `{id}` / `channel_sync_status` | a channel's Discord name and topic / each synced chat channel's last result (above) |
|
||||
| `notes_list` / `note_update` `{id, text}` / `note_delete` `{id}` | everyone's personal notes (above) / change one's text / delete one |
|
||||
|
||||
@@ -311,7 +311,15 @@ impl AdminApi {
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(json!({"saved": changes.len(), "restartNeeded": self.controller()?.state() == crate::control::State::Running,
|
||||
// The speech speed reaches the running services at once; everything else waits for
|
||||
// a restart.
|
||||
if changes.contains_key("voice.tts.speed") {
|
||||
let speed = crate::settings::load_settings(&file).map_err(|e| e.0)?.voice.tts_speed;
|
||||
self.controller()?.set_tts_speed(speed);
|
||||
}
|
||||
let restart = changes.keys().any(|k| k != "voice.tts.speed");
|
||||
Ok(json!({"saved": changes.len(),
|
||||
"restartNeeded": restart && self.controller()?.state() == crate::control::State::Running,
|
||||
"sync": sync}))
|
||||
}
|
||||
"mcp_env_status" => {
|
||||
|
||||
@@ -30,6 +30,8 @@ pub trait Running: Send + Sync + 'static {
|
||||
async fn stop(&self) -> Result<(), String>;
|
||||
/// Worker, voice channel members and missed requests, for the snapshot.
|
||||
fn status(&self, now: Instant) -> Value;
|
||||
/// Speak at this speed (`voice.tts.speed`) from the next sentence on.
|
||||
fn set_tts_speed(&self, _speed: f64) {}
|
||||
}
|
||||
|
||||
/// What starts the services (the real ones, or a fake in tests).
|
||||
@@ -167,6 +169,13 @@ impl Controller {
|
||||
Err("cancelled".into())
|
||||
}
|
||||
|
||||
/// Give the running services a new speech speed. False when nothing runs (the next start reads it
|
||||
/// from the settings).
|
||||
pub fn set_tts_speed(&self, speed: f64) -> bool {
|
||||
let running = self.inner.lock().unwrap().running.clone();
|
||||
running.map(|r| r.set_tts_speed(speed)).is_some()
|
||||
}
|
||||
|
||||
/// Stop from running, or cancel a start; what was said is forgotten.
|
||||
pub async fn stop(&self) -> Result<(), String> {
|
||||
let running = {
|
||||
@@ -367,6 +376,10 @@ impl Running for crate::services::Services {
|
||||
"performance": self.llm.stage_view(),
|
||||
})
|
||||
}
|
||||
|
||||
fn set_tts_speed(&self, speed: f64) {
|
||||
self.worker.set_tts_speed(speed);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -381,6 +394,7 @@ mod tests {
|
||||
stop_fails: std::sync::atomic::AtomicBool,
|
||||
cleanups: AtomicU32,
|
||||
cleanup_fails: std::sync::atomic::AtomicBool,
|
||||
speeds: Mutex<Vec<f64>>,
|
||||
}
|
||||
|
||||
struct FakeRunning(Arc<Counts>);
|
||||
@@ -393,6 +407,9 @@ mod tests {
|
||||
fn status(&self, _: Instant) -> Value {
|
||||
json!({"worker": {"connected": true}, "voiceMembers": [{"id": "11", "name": "민수"}], "missed": []})
|
||||
}
|
||||
fn set_tts_speed(&self, speed: f64) {
|
||||
self.0.speeds.lock().unwrap().push(speed);
|
||||
}
|
||||
}
|
||||
|
||||
struct FakeHost {
|
||||
@@ -471,6 +488,17 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_new_speech_speed_reaches_the_running_services_without_a_restart() {
|
||||
let counts = Arc::new(Counts::default());
|
||||
let c = Controller::new(FakeHost { counts: counts.clone(), fail_first: AtomicU32::new(0), events: Mutex::new(None) });
|
||||
assert!(!c.set_tts_speed(1.7), "nothing runs: the next start reads the settings");
|
||||
c.start().await.unwrap();
|
||||
assert!(c.set_tts_speed(1.8));
|
||||
assert_eq!(*counts.speeds.lock().unwrap(), [1.8]);
|
||||
assert_eq!(counts.starts.load(Ordering::SeqCst), 1, "no restart");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_stop_while_starting_cancels_the_start_or_stops_what_came_up() {
|
||||
for honours_cancel in [true, false] {
|
||||
|
||||
@@ -209,6 +209,8 @@ struct Inner {
|
||||
activity: OnceLock<Arc<crate::activity::Activity>>,
|
||||
/// Speech requests whose end someone waits for (`say_to_end`), by request id.
|
||||
speech_waits: Mutex<HashMap<String, tokio::sync::oneshot::Sender<PlaybackEnd>>>,
|
||||
/// The spoken transcription notice of the current voice session.
|
||||
notice: Mutex<JoinNotice>,
|
||||
}
|
||||
|
||||
/// How long `say_to_end` waits for a reminder's speech (behind what is queued before it).
|
||||
@@ -243,6 +245,55 @@ impl UttSessions {
|
||||
}
|
||||
}
|
||||
|
||||
/// The transcription notice (`voice.joinNotice`) is spoken in the voice channel: it is owed from each
|
||||
/// join until some of it was played or someone stopped it. Speech that could not be made (the worker
|
||||
/// not connected yet) leaves it owed, and it is spoken when the worker is next ready.
|
||||
#[derive(Default)]
|
||||
pub struct JoinNotice {
|
||||
owed: bool,
|
||||
speaking: Option<String>,
|
||||
}
|
||||
|
||||
impl JoinNotice {
|
||||
pub fn joined(&mut self) {
|
||||
*self = Self { owed: true, speaking: None };
|
||||
}
|
||||
pub fn left(&mut self) {
|
||||
*self = Self::default();
|
||||
}
|
||||
/// It should be spoken now (owed and not already in the queue).
|
||||
pub fn due(&self) -> bool {
|
||||
self.owed && self.speaking.is_none()
|
||||
}
|
||||
/// It was queued as speech request `id`.
|
||||
pub fn queued(&mut self, id: String) {
|
||||
self.speaking = Some(id);
|
||||
}
|
||||
/// A speech request ended. Some(heard) when it was the notice: false means it is still owed.
|
||||
pub fn ended(&mut self, e: &PlaybackEnd) -> Option<bool> {
|
||||
if self.speaking.as_deref() != Some(e.id.as_str()) {
|
||||
return None;
|
||||
}
|
||||
self.speaking = None;
|
||||
let heard = e.played > 0 || e.cancelled;
|
||||
self.owed = !heard;
|
||||
Some(heard)
|
||||
}
|
||||
}
|
||||
|
||||
/// Queue the transcription notice when it is owed and the worker can make speech.
|
||||
fn speak_notice(inner: &Inner) {
|
||||
if inner.session.lock().unwrap().is_none() || !inner.worker.is_ready() {
|
||||
return;
|
||||
}
|
||||
with_playback(inner, |p| {
|
||||
let mut n = inner.notice.lock().unwrap();
|
||||
if n.due() {
|
||||
n.queued(p.say(&inner.settings.voice.join_notice, false));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Handle to the core; cheap to clone.
|
||||
#[derive(Clone)]
|
||||
pub struct DiscordCore {
|
||||
@@ -280,6 +331,7 @@ impl DiscordCore {
|
||||
schedules: OnceLock::new(),
|
||||
tasks: Mutex::new(HashMap::new()),
|
||||
speech_waits: Mutex::new(HashMap::new()),
|
||||
notice: Mutex::new(JoinNotice::default()),
|
||||
activity: OnceLock::new(),
|
||||
notes: OnceLock::new(),
|
||||
});
|
||||
@@ -398,7 +450,7 @@ impl DiscordCore {
|
||||
self.inner.session.lock().unwrap().as_ref().map(|s| s.channel_id)
|
||||
}
|
||||
|
||||
/// Join a voice channel of the configured server (leaving the current one) and post the notice.
|
||||
/// Join a voice channel of the configured server (leaving the current one) and speak the notice.
|
||||
pub async fn join(&self, channel_id: u64) -> Result<(), CoreError> {
|
||||
let _guard = self.inner.voice_lock.lock().await;
|
||||
let http = self.http()?.clone();
|
||||
@@ -455,11 +507,8 @@ impl DiscordCore {
|
||||
report_voice_state(&self.inner);
|
||||
debug_log!("voice", "joined voice {channel_id}");
|
||||
let _ = self.inner.events.send(CoreEvent::Joined { guild_id: gc.guild_id.get(), channel_id, session });
|
||||
let notice = CreateMessage::new().content(&self.inner.settings.voice.join_notice)
|
||||
.allowed_mentions(CreateAllowedMentions::new());
|
||||
if let Err(e) = gc.id.send_message(&http, notice).await {
|
||||
error_log!("voice", "could not post the transcription notice in {channel_id}: {e}");
|
||||
}
|
||||
self.inner.notice.lock().unwrap().joined();
|
||||
speak_notice(&self.inner);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -736,6 +785,12 @@ fn with_playback<T>(inner: &Inner, f: impl FnOnce(&mut Playback<WorkerClient, Pl
|
||||
if let Some(tx) = inner.speech_waits.lock().unwrap().remove(&end.id) {
|
||||
let _ = tx.send(end.clone());
|
||||
}
|
||||
match inner.notice.lock().unwrap().ended(end) {
|
||||
Some(true) => { debug_log!("voice", "transcription notice spoken id={} played={}", end.id, end.played); }
|
||||
Some(false) => { error_log!("voice", "could not speak the transcription notice; it is spoken when \
|
||||
the voice worker is next ready"); }
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
let _ = inner.events.send(match e {
|
||||
PlaybackEvent::Start(id) => CoreEvent::SpeakStart(id),
|
||||
@@ -753,6 +808,7 @@ async fn end_session(inner: &Arc<Inner>, s: Session, reason: &str, disconnect: b
|
||||
inner.utt_sessions.lock().unwrap().clear();
|
||||
let _ = inner.events.send(CoreEvent::Stopped);
|
||||
with_playback(inner, |p| p.stop());
|
||||
inner.notice.lock().unwrap().left();
|
||||
let _ = inner.player.send(PlayerCmd::SetCall(None));
|
||||
let (heard, lost, late, bad) = {
|
||||
let mut rx = s.rx.lock().unwrap();
|
||||
@@ -898,6 +954,7 @@ async fn worker_task(core: std::sync::Weak<Inner>, mut rx: mpsc::UnboundedReceiv
|
||||
WorkerEvent::Ready(_) => {
|
||||
report_chat_channels(&inner);
|
||||
report_voice_state(&inner);
|
||||
speak_notice(&inner);
|
||||
}
|
||||
WorkerEvent::ChatAction(a) => {
|
||||
let core = DiscordCore { inner: inner.clone() };
|
||||
@@ -1350,6 +1407,41 @@ mod tests {
|
||||
assert!(!with_playback(&core.inner, |p| p.busy()), "nothing is left in the queue");
|
||||
}
|
||||
|
||||
fn end(id: &str, played: u32, cancelled: bool, failed: bool) -> PlaybackEnd {
|
||||
PlaybackEnd { id: id.into(), played, cancelled, failed }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_notice_is_owed_from_each_join_until_it_was_heard() {
|
||||
let mut n = super::JoinNotice::default();
|
||||
assert!(!n.due(), "nothing is owed outside a voice channel");
|
||||
n.joined();
|
||||
assert!(n.due());
|
||||
n.queued("r1".into());
|
||||
assert!(!n.due(), "queued once, not twice");
|
||||
assert_eq!(n.ended(&end("r2", 1, false, false)), None, "other speech does not settle it");
|
||||
assert_eq!(n.ended(&end("r1", 2, false, false)), Some(true));
|
||||
assert!(!n.due(), "heard: not spoken again, also after a worker restart");
|
||||
n.joined();
|
||||
assert!(n.due(), "every join owes it again");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_notice_the_worker_could_not_speak_stays_owed() {
|
||||
let mut n = super::JoinNotice::default();
|
||||
n.joined();
|
||||
n.queued("r1".into());
|
||||
assert_eq!(n.ended(&end("r1", 0, false, true)), Some(false));
|
||||
assert!(n.due(), "spoken when the worker is next ready");
|
||||
// Someone stopped it: they were in the channel when it started, it is not forced on them again.
|
||||
n.queued("r2".into());
|
||||
assert_eq!(n.ended(&end("r2", 0, true, false)), Some(true));
|
||||
assert!(!n.due());
|
||||
n.joined();
|
||||
n.left();
|
||||
assert!(!n.due(), "leaving drops it");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_utterance_keeps_the_session_it_started_in() {
|
||||
let mut u = super::UttSessions::default();
|
||||
|
||||
@@ -11,8 +11,8 @@ worker side of the link is `src/voice_worker/voice_worker.spec.md`.
|
||||
## Principles
|
||||
|
||||
- **Only where the bot is.** The bot hears only the voice channel it is in and reads only the chat
|
||||
channels registered in settings with reading on. Every join posts the transcription notice in the
|
||||
voice channel's chat.
|
||||
channels registered in settings with reading on. Every join speaks the transcription notice in the
|
||||
voice channel.
|
||||
- **Chat is data.** Messages read from chat channels are handed on as data, never acted on as
|
||||
commands. Text the bot sends never pings anyone (empty allowed mentions).
|
||||
- **No text in logs.** Logs carry ids, sizes, counts and timings, never what anyone said or wrote, or
|
||||
@@ -66,11 +66,11 @@ strings (numbers would lose precision).
|
||||
| `discord.ephemeralReplies` | `true` | Command replies only the caller sees. |
|
||||
| `voice.maxSpeakers` | `10` | Most people transcribed at once, 1-10. |
|
||||
| `voice.listenToBots` | `false` | Also transcribe other bots (music bots are ignored by default). |
|
||||
| `voice.joinNotice` | "자비스가 이 음성 채널에 들어왔습니다. 이 채널의 대화는 자비스가 받아 적습니다." | Posted in the voice channel's chat on every join. Must not be empty. |
|
||||
| `voice.joinNotice` | "자비스가 이 음성 채널에 들어왔습니다. 이 채널의 대화는 자비스가 받아 적습니다." | Spoken in the voice channel on every join. Must not be empty. |
|
||||
| `voice.wakeWords` | `["자비스"]` | Sent to the worker as STT `hotwords`, and the voice wake words of call detection. |
|
||||
| `voice.stt` | `{}` | Other worker STT settings (`partials`, `end_silence_ms`, `final_beam`, `language`, ...); the worker validates them. |
|
||||
| `chatChannels` | `[]` | `{id, name, description, read (true), send (true), edit (true), react (true), delete (false), pin (false), answer ("wakeWord"), syncDiscord (true)}` per channel; `answer` is in `conversation.spec.md`, `syncDiscord` (the channel's Discord name and topic follow `name` and `description`) in `admin.spec.md`. |
|
||||
| `voice.tts.speed` | `1.5` | How fast the voice speaks (MeloTTS `speed`, 0.5-2.0), passed to the worker as `--melo-speed` when it speaks (`--melo-python`) and its arguments set no speed. MeloTTS's Korean model has one voice, so there is no voice choice. |
|
||||
| `voice.tts.speed` | `1.5` | How fast the voice speaks, 0.5-2.0 (1.0 is MeloTTS's natural pace; above it the worker stretches the speech and keeps its pitch, `voice_worker.spec.md`, "TTS"). Passed to the worker as `--melo-speed` when it speaks (`--melo-python`) and its arguments set no speed. A change saved while the services run goes to the worker at once (`tts_speed`) and applies from the next sentence, without a restart; it is sent again whenever the worker connects again. MeloTTS's Korean model has one voice, so there is no voice choice. |
|
||||
| `voiceWorker.python` | `python3` (`python.exe` on Windows) | Python of the worker venv. |
|
||||
| `voiceWorker.cwd` | `python-src` next to the app's executable when it is there (the Windows build), else the repo's `src/` | Folder holding the `voice_worker` package. |
|
||||
| `voiceWorker.args` | `[]` | Extra worker arguments, e.g. `["--melo-python", "<MeloTTS venv python>"]`. |
|
||||
@@ -126,8 +126,12 @@ the new channel and its event handlers are replaced: leaving and joining at once
|
||||
and the late "left" update cancelled the new join (seen on real Discord).
|
||||
|
||||
- **Join** (slash command `join`, auto-join, or `join(channel_id)`): connect through songbird with DAVE
|
||||
end-to-end encryption, not deafened, then post `voice.joinNotice` in the voice channel's chat. A
|
||||
notice that cannot be posted (no permission) is logged and the bot stays.
|
||||
end-to-end encryption, not deafened, then speak `voice.joinNotice` in the voice channel (a speech
|
||||
request like any other, nothing is posted in chat). The notice is owed from the join until some of it
|
||||
was played or someone stopped it. When it cannot be spoken (the worker is not connected yet, as at
|
||||
auto-join while the worker still starts), it stays owed and is spoken as soon as the worker is ready;
|
||||
this is logged as an error and the bot stays. Leaving drops it; a worker restart later in the session
|
||||
does not repeat a notice that was heard.
|
||||
- **Hearing**: songbird only decrypts (transport and DAVE). Its own Opus decoding is not used: in
|
||||
songbird 0.6.1 it decodes the payload without trimming what DAVE decryption left at its end, which
|
||||
garbled, repeated or dropped about a third of the frames (measured with a test tone that changes
|
||||
|
||||
@@ -120,7 +120,7 @@ pub fn fields() -> Vec<Field> {
|
||||
f("voice.stt.final_beam", Model, Int { min: 1, max: 5 }, json!(2), "최종 전사 정확도(빔)"),
|
||||
f("voice.stt.language", Model, LanguageCode, json!("ko"), "받아 적을 언어(비우면 자동)"),
|
||||
// MeloTTS's Korean model has one voice (speaker "KR"), so only the speed is a choice.
|
||||
f("voice.tts.speed", Model, Number { min: 0.5, max: 2.0 }, json!(1.5), "말하기 속도(MeloTTS, 기본 1.5)"),
|
||||
f("voice.tts.speed", Model, Number { min: 0.5, max: 2.0 }, json!(1.5), "말하기 속도(1.0 보통, 기본 1.5, 저장하면 바로 적용)"),
|
||||
// The LLM (admin.spec.md, "LLM"); `llm.url` (a server the app does not start) is file-only.
|
||||
f("llm.context", Model, Int { min: 4096, max: 262144 }, json!(32768), "대화 모델이 한 번에 기억하는 양(토큰)"),
|
||||
f("llm.port", Model, Int { min: 1024, max: 65535 }, json!(8080), "대화 모델 포트"),
|
||||
|
||||
@@ -226,6 +226,9 @@ struct Inner {
|
||||
/// The LLM state the brain should have: sent again on every connection, so a worker that
|
||||
/// missed it (a dropped connection, a restarted worker) catches up.
|
||||
llm: Mutex<Option<Value>>,
|
||||
/// The latest speech speed set while running, sent again on every connection: a restarted worker
|
||||
/// starts with the speed of its arguments.
|
||||
tts_speed: Mutex<Option<f64>>,
|
||||
/// The number of the latest switch.
|
||||
llm_switches: std::sync::atomic::AtomicU64,
|
||||
}
|
||||
@@ -248,6 +251,7 @@ impl WorkerClient {
|
||||
wake: Notify::new(),
|
||||
llm_idle: watch::channel(0).0,
|
||||
llm: Mutex::new(None),
|
||||
tts_speed: Mutex::new(None),
|
||||
llm_switches: std::sync::atomic::AtomicU64::new(0),
|
||||
});
|
||||
(Self { inner }, rx)
|
||||
@@ -497,6 +501,14 @@ impl WorkerClient {
|
||||
self.set_llm(json!({"type": "llm", "state": "ready", "game": game}))
|
||||
}
|
||||
|
||||
/// Speak at `speed` from the next sentence on (`tts_speed`). False when not connected; it still
|
||||
/// goes out on the next connection.
|
||||
pub fn set_tts_speed(&self, speed: f64) -> bool {
|
||||
let mut s = self.inner.tts_speed.lock().unwrap();
|
||||
*s = Some(speed);
|
||||
self.send_json(json!({"type": "tts_speed", "speed": speed}))
|
||||
}
|
||||
|
||||
fn set_llm(&self, m: Value) -> bool {
|
||||
let mut llm = self.inner.llm.lock().unwrap();
|
||||
*llm = Some(m.clone());
|
||||
@@ -637,9 +649,11 @@ async fn connect_once(inner: &Arc<Inner>) -> bool {
|
||||
|
||||
let (tx, mut rx) = mpsc::unbounded_channel::<Message>();
|
||||
{
|
||||
// Locks in the order llm, then state (as `llm_state` takes them): the latest LLM state goes
|
||||
// first, and the connection is ready for sending before an LLM state set meanwhile is sent.
|
||||
// Locks in the order llm, tts_speed, then state (as `llm_state` and `set_tts_speed` take
|
||||
// them): the latest LLM state and speed go first, and the connection is ready for sending
|
||||
// before one set meanwhile is sent.
|
||||
let llm = inner.llm.lock().unwrap();
|
||||
let speed = inner.tts_speed.lock().unwrap();
|
||||
let mut st = inner.state.lock().unwrap();
|
||||
if st.closed {
|
||||
return true;
|
||||
@@ -647,6 +661,9 @@ async fn connect_once(inner: &Arc<Inner>) -> bool {
|
||||
if let Some(m) = llm.clone() {
|
||||
let _ = tx.send(Message::text(m.to_string()));
|
||||
}
|
||||
if let Some(s) = *speed {
|
||||
let _ = tx.send(Message::text(json!({"type": "tts_speed", "speed": s}).to_string()));
|
||||
}
|
||||
st.conn = Some(tx);
|
||||
inner.ready.send_replace(Some(hello_ok.clone()));
|
||||
}
|
||||
|
||||
@@ -198,9 +198,13 @@ async fn discord_voice_core_on_real_discord() {
|
||||
ev2.lock().unwrap().push(e);
|
||||
}
|
||||
});
|
||||
core.start().await.expect("core start"); // logs in, registers the command, auto-joins, posts the notice
|
||||
core.start().await.expect("core start"); // logs in, registers the command, auto-joins, speaks the notice
|
||||
assert_eq!(core.voice_channel_id(), Some(voice));
|
||||
let jarvis = core.bot_id().unwrap();
|
||||
// The transcription notice is the first speech of the join, and it is spoken, not posted.
|
||||
until("notice spoken", 60_000, || events.lock().unwrap().iter().any(|e| matches!(e, CoreEvent::SpeakEnd(_)))).await;
|
||||
let notice = events.lock().unwrap().iter().find_map(|e| match e { CoreEvent::SpeakEnd(x) => Some(x.clone()), _ => None }).unwrap();
|
||||
assert!(!notice.failed && !notice.cancelled && notice.played > 0, "{notice:?}");
|
||||
|
||||
let call = speaker_songbird.join(GuildId::new(guild), ChannelId::new(voice)).await.expect("speaker joins");
|
||||
let ears = Ears { jarvis, ..Default::default() };
|
||||
@@ -320,7 +324,7 @@ async fn discord_voice_core_on_real_discord() {
|
||||
assert!(tail < 400, "audio went on {tail} ms after stop");
|
||||
assert!(!core.speaking());
|
||||
|
||||
// 5. Chat: live event, history with the join notice, and a send that pings nobody.
|
||||
// 5. Chat: live event, history, and a send that pings nobody.
|
||||
let marker = format!("라이브 테스트 메시지 {}", stop_at.elapsed().as_nanos());
|
||||
ChannelId::new(voice).send_message(&s_http, CreateMessage::new().content(&marker)).await.unwrap();
|
||||
until("chat event", 10_000, || events.lock().unwrap().iter().any(|e| matches!(e, CoreEvent::Chat(c) if c.content == marker))).await;
|
||||
@@ -328,7 +332,6 @@ async fn discord_voice_core_on_real_discord() {
|
||||
assert!(ev.channel_id == voice && ev.author_id == speaker_id && ev.author_is_bot && !ev.from_self);
|
||||
let history = core.read_chat(voice, 20).await.unwrap();
|
||||
assert!(history.iter().any(|m| m.content == marker && !m.from_self));
|
||||
assert!(history.iter().any(|m| m.from_self && m.content.contains("받아 적습니다")), "join notice missing");
|
||||
// An edit and a delete reach the history store.
|
||||
let mut marker_msg = ChannelId::new(voice).messages(&s_http, serenity::all::GetMessages::new().limit(5)).await.unwrap()
|
||||
.into_iter().find(|m| m.content == marker).unwrap();
|
||||
|
||||
@@ -503,3 +503,22 @@ async fn ready_tells_the_worker_whether_a_game_runs() {
|
||||
until(|| w.got().iter().any(|g| *g == Got::Json(json!({"type": "llm", "state": "ready", "game": true})))).await;
|
||||
c.close();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_speech_speed_goes_out_at_once_and_again_on_every_connection() {
|
||||
let w = FakeWorker::start(0).await;
|
||||
let (c, mut rx) = client(&w.url(), json!({}));
|
||||
c.start();
|
||||
c.wait_ready(Duration::from_secs(2)).await.unwrap();
|
||||
assert!(c.set_tts_speed(1.8), "sent while connected");
|
||||
until(|| w.got().contains(&Got::Json(json!({"type": "tts_speed", "speed": 1.8})))).await;
|
||||
// A restarted worker starts with the speed of its arguments; the latest one set catches it up.
|
||||
let port = w.kill();
|
||||
next_matching(&mut rx, |e| *e == WorkerEvent::Disconnected).await;
|
||||
assert!(!c.set_tts_speed(1.2), "not sent: no connection");
|
||||
let w2 = FakeWorker::start(port).await;
|
||||
until(|| w2.got().iter().any(|g| matches!(g, Got::Json(v) if v["type"] == "tts_speed"))).await;
|
||||
let speeds: Vec<_> = w2.got().into_iter().filter(|g| matches!(g, Got::Json(v) if v["type"] == "tts_speed")).collect();
|
||||
assert_eq!(speeds, [Got::Json(json!({"type": "tts_speed", "speed": 1.2}))]);
|
||||
c.close();
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# 개인정보·법적 안전장치
|
||||
|
||||
- 모든 처리는 게임PC 안에서만 합니다. 클라우드 API를 쓰지 않습니다. 밖으로 나가는 것은 웹 검색과 날씨 조회 요청뿐입니다.
|
||||
- 봇이 음성 채널에 들어가면 "대화가 전사됩니다" 고지 메시지를 보냅니다. 문구는 설정에서 수정합니다.
|
||||
- 봇이 음성 채널에 들어가면 "대화가 전사됩니다" 고지를 음성으로 말합니다(채팅에는 올리지 않습니다). 음성 워커가 아직 준비 중이면 준비되는 대로 말합니다. 문구는 설정에서 수정합니다.
|
||||
- 봇이 참여한 음성 채널의 대화만 처리합니다. 봇이 없는 대화를 몰래 듣는 기능은 만들지 않습니다.
|
||||
- 채팅은 GUI에 등록한 채팅 채널만 읽고 저장합니다. 그 밖의 채널은 읽지 않습니다. 채팅에 올라온 글은 명령이 아니라 데이터로 다룹니다.
|
||||
- 음성 원본은 기본적으로 저장하지 않습니다. 디버그용 저장은 직접 켰을 때만 하고, 기간이 지나면 자동 삭제합니다.
|
||||
|
||||
@@ -2,11 +2,18 @@
|
||||
|
||||
Stand-alone on purpose: it must not import the worker package, which lives in another venv.
|
||||
|
||||
stdin: one JSON request per line: {"text": "...", "rate": 48000}
|
||||
stdin: one JSON request per line: {"text": "...", "rate": 48000, "speed": 1.5} (speed optional,
|
||||
default --speed)
|
||||
stdout: frames of u32 LE header length | header JSON | payload. The first frame is
|
||||
{"ready": true, "rate": <native rate>}; each reply is {"ok": true, "samples": N} followed by N
|
||||
int16 LE mono samples at the requested rate, or {"ok": false, "error": "..."} with no payload.
|
||||
Library output that would land on stdout is sent to stderr so it cannot corrupt the frames.
|
||||
|
||||
Speed: MeloTTS's own speed shortens every sound and slurs faster speech (Whisper large-v3 on five
|
||||
Korean sentences, three renders each: character errors 5.5% at 1.5 and 20% at 2.0, against 1.4% at
|
||||
1.0). So the model speaks at most at MODEL_SPEED_MAX (its natural pace) and a time stretch that keeps
|
||||
the pitch (WSOLA) makes it faster: 1.9% at 1.5 and 2.2% at 2.0. Letting the model go to 1.25 first
|
||||
measured slightly worse (2.2% and 3.1%).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -19,6 +26,42 @@ import struct
|
||||
import sys
|
||||
import types
|
||||
|
||||
MODEL_SPEED_MAX = 1.0
|
||||
|
||||
|
||||
def split_speed(speed: float) -> tuple[float, float]:
|
||||
"""(the model's speed, the stretch factor after it); their product is ``speed``."""
|
||||
model = min(speed, MODEL_SPEED_MAX)
|
||||
return model, speed / model
|
||||
|
||||
|
||||
def stretch(audio, factor: float, rate: int, frame_ms: float = 30.0, search_ms: float = 10.0):
|
||||
"""Shorten ``audio`` by ``factor`` and keep its pitch (WSOLA): half-overlapping Hann frames are
|
||||
taken ``factor`` times further apart than they are laid down, each shifted within ``search_ms``
|
||||
to where it best continues the frame before it, so the pieces join without dips or clicks."""
|
||||
import numpy as np
|
||||
|
||||
x = np.asarray(audio, dtype=np.float64)
|
||||
if abs(factor - 1.0) < 1e-3 or len(x) == 0:
|
||||
return x
|
||||
n = int(rate * frame_ms / 1000) // 2 * 2
|
||||
hop = n // 2
|
||||
tol = int(rate * search_ms / 1000)
|
||||
win = 0.5 - 0.5 * np.cos(2 * np.pi * np.arange(n) / n) # its half-overlapped copies sum to 1
|
||||
n_out = int(np.ceil(len(x) / factor))
|
||||
frames = n_out // hop + 2
|
||||
xp = np.pad(x, (tol, int(frames * hop * factor) + 2 * n + 2 * tol))
|
||||
y = np.zeros(frames * hop + n)
|
||||
prev = None
|
||||
for k in range(frames):
|
||||
pos = int(round(k * hop * factor)) + tol
|
||||
if prev is not None:
|
||||
natural = xp[prev + hop: prev + hop + n]
|
||||
pos += int(np.argmax(np.correlate(xp[pos - tol: pos + tol + n], natural, mode="valid"))) - tol
|
||||
y[k * hop: k * hop + n] += xp[pos: pos + n] * win
|
||||
prev = pos
|
||||
return y[:n_out]
|
||||
|
||||
|
||||
def korean_only_patches() -> None:
|
||||
"""MeloTTS imports its Japanese front end (module-level MeCab.Tagger()) even for Korean, and on
|
||||
@@ -76,7 +119,7 @@ def main() -> int:
|
||||
spk_map = model.hps.data.spk2id
|
||||
speaker = spk_map[args.language] if args.language in spk_map else spk_map[list(spk_map.keys())[0]]
|
||||
native = int(model.hps.data.sampling_rate)
|
||||
model.tts_to_file("워밍업입니다.", speaker, None, speed=args.speed, quiet=True)
|
||||
model.tts_to_file("워밍업입니다.", speaker, None, speed=split_speed(args.speed)[0], quiet=True)
|
||||
except Exception as e:
|
||||
reply({"ready": False, "error": f"{type(e).__name__}: {e}"})
|
||||
return 1
|
||||
@@ -86,8 +129,9 @@ def main() -> int:
|
||||
try:
|
||||
req = json.loads(line)
|
||||
rate = int(req["rate"])
|
||||
audio = model.tts_to_file(str(req["text"]), speaker, None, speed=args.speed, quiet=True)
|
||||
audio = np.asarray(audio, dtype=np.float64)
|
||||
model_speed, factor = split_speed(float(req.get("speed", args.speed)))
|
||||
audio = model.tts_to_file(str(req["text"]), speaker, None, speed=model_speed, quiet=True)
|
||||
audio = stretch(audio, factor, native)
|
||||
if rate != native:
|
||||
from math import gcd
|
||||
g = gcd(rate, native)
|
||||
|
||||
@@ -346,6 +346,7 @@ class Session:
|
||||
kind = msg.get("type")
|
||||
handler = {"speaking_end": self._on_speaking_end, "speaker_leave": self._on_speaker_leave,
|
||||
"config": self._on_config, "tts": self._on_tts, "tts_cancel": self._on_tts_cancel,
|
||||
"tts_speed": self._on_tts_speed,
|
||||
"voice_join": self._on_voice_join, "voice_leave": lambda m: self._leave_voice(),
|
||||
"member_join": self._on_member_join, "member_leave": self._on_member_leave,
|
||||
"chat_message": self._on_chat_message, "chat_edit": self._on_chat_edit,
|
||||
@@ -388,6 +389,15 @@ class Session:
|
||||
self.worker.scheduler.drop_partials(lambda key: key[0] == self.sid)
|
||||
debug_log(f"session {self.sid} settings changed: {sorted(changes)}", "voice_worker")
|
||||
|
||||
def _on_tts_speed(self, msg: dict) -> None:
|
||||
speed = msg.get("speed")
|
||||
if isinstance(speed, bool) or not isinstance(speed, (int, float)) or not 0.5 <= speed <= 2.0:
|
||||
self.error("bad_message", "tts_speed needs a speed from 0.5 to 2.0")
|
||||
return
|
||||
if self.worker.tts_engine is not None:
|
||||
self.worker.tts_engine.speed = float(speed)
|
||||
debug_log(f"tts speed {speed}", "tts")
|
||||
|
||||
def _on_tts(self, msg: dict) -> None:
|
||||
rid, text, more = msg.get("id"), msg.get("text", ""), msg.get("more", False)
|
||||
if not isinstance(rid, str) or not 0 < len(rid.encode()) < 256 or not isinstance(text, str) \
|
||||
|
||||
@@ -19,6 +19,8 @@ CHILD = os.path.join(os.path.dirname(os.path.abspath(__file__)), "melo_child.py"
|
||||
class MeloEngine:
|
||||
def __init__(self, python: str, device: str = "cuda", speed: float = 1.5, language: str = "KR"):
|
||||
self.cmd = [python, CHILD, "--device", device, "--speed", str(speed), "--language", language]
|
||||
# Read for each sentence, so a new speed (`tts_speed`) applies from the next one.
|
||||
self.speed = speed
|
||||
self._lock = threading.Lock()
|
||||
self._proc: subprocess.Popen | None = None
|
||||
self.native_rate: int | None = None
|
||||
@@ -49,7 +51,7 @@ class MeloEngine:
|
||||
if self._proc is None or self._proc.poll() is not None:
|
||||
debug_log("melo child not running, restarting", "tts")
|
||||
self._start()
|
||||
self._proc.stdin.write((json.dumps({"text": text, "rate": rate}) + "\n").encode())
|
||||
self._proc.stdin.write((json.dumps({"text": text, "rate": rate, "speed": self.speed}) + "\n").encode())
|
||||
self._proc.stdin.flush()
|
||||
header, payload = self._read()
|
||||
if not header.get("ok"):
|
||||
|
||||
@@ -164,6 +164,7 @@ line (-0.6) would lose more real words than it saves.
|
||||
```
|
||||
-> {"type": "tts", "id": "r1", "text": "...", "more": false} // more=true: more text follows for r1
|
||||
-> {"type": "tts_cancel", "id": "r1"}
|
||||
-> {"type": "tts_speed", "speed": 1.8} // 0.5-2.0, from the next sentence on
|
||||
<- binary 0x02 | u8 n | id (n bytes UTF-8) | u32 LE seq | PCM16LE in the hello "tts" format
|
||||
<- {"type": "tts_done", "id": "r1", "chunks": 3, "cancelled": false, "failed": false}
|
||||
```
|
||||
@@ -178,6 +179,14 @@ is ready. Requests are served first come, first served by one TTS engine. `tts_c
|
||||
request's unsent sentences, and `tts_done` follows with `cancelled: true`. A sentence that fails to
|
||||
synthesise is reported with `error` (`code: "tts_failed"`) and the request ends with `failed: true`.
|
||||
|
||||
Speed starts at `--melo-speed` and `tts_speed` changes it for every sentence made after it (a speed
|
||||
outside 0.5-2.0 or not a number is `bad_message` and the last good one stays). MeloTTS itself speaks
|
||||
at most at its natural pace (1.0): its own speed shortens every sound and slurs faster speech
|
||||
(Whisper large-v3 on five Korean sentences, three renders each: 5.5% character errors at 1.5 and 20%
|
||||
at 2.0, against 1.4% at 1.0). Faster speech is made at 1.0 and shortened with WSOLA, a time stretch
|
||||
that keeps the pitch (30 ms half-overlapping Hann frames, each shifted within 10 ms to where it best
|
||||
continues the one before): 1.9% at 1.5 and 2.2% at 2.0, about 15 ms per 4 s of speech.
|
||||
|
||||
### History and brain (M3)
|
||||
|
||||
The worker also hosts the brain (the existing `jarvis` reply engine) and the conversation history
|
||||
|
||||
89
tests/test_voice_worker_melo_speed.py
Normal file
89
tests/test_voice_worker_melo_speed.py
Normal file
@@ -0,0 +1,89 @@
|
||||
"""How the MeloTTS child speeds speech up: the model never speaks faster than MODEL_SPEED_MAX (faster
|
||||
slurs), and the rest is a time stretch that keeps the pitch. And the engine sends its current speed
|
||||
with every sentence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from voice_worker import melo_child, tts_melo
|
||||
from voice_worker.melo_child import MODEL_SPEED_MAX, split_speed, stretch
|
||||
|
||||
SR = 44100
|
||||
|
||||
|
||||
def sine(hz: float, seconds: float) -> np.ndarray:
|
||||
return np.sin(2 * np.pi * hz * np.arange(int(SR * seconds)) / SR)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_model_never_speaks_faster_than_its_limit_and_the_stretch_does_the_rest():
|
||||
for speed in (0.5, 0.8, MODEL_SPEED_MAX):
|
||||
assert split_speed(speed) == (speed, 1.0)
|
||||
for speed in (1.5, 1.75, 2.0):
|
||||
model, factor = split_speed(speed)
|
||||
assert model == MODEL_SPEED_MAX and factor > 1.0
|
||||
assert model * factor == pytest.approx(speed)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_stretch_shortens_by_the_factor_and_keeps_the_pitch():
|
||||
x = sine(220.0, 2.0)
|
||||
for factor in (1.2, 1.6, 2.0):
|
||||
y = stretch(x, factor, SR)
|
||||
assert len(y) == pytest.approx(len(x) / factor, abs=2)
|
||||
spectrum = np.abs(np.fft.rfft(y * np.hanning(len(y))))
|
||||
assert np.argmax(spectrum) * SR / len(y) == pytest.approx(220.0, abs=2.0)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_stretched_sound_stays_even_without_dips_or_clicks():
|
||||
# Badly joined pieces cancel each other (the level dips) or jump (clicks).
|
||||
x = 0.5 * sine(180.0, 1.5)
|
||||
y = stretch(x, 1.6, SR)
|
||||
middle = y[SR // 10: -SR // 10]
|
||||
frame = SR // 100
|
||||
levels = [np.sqrt(np.mean(middle[i:i + frame] ** 2)) for i in range(0, len(middle) - frame, frame)]
|
||||
assert min(levels) > 0.5 / np.sqrt(2) * 0.9 and max(levels) < 0.5 / np.sqrt(2) * 1.1
|
||||
assert np.max(np.abs(np.diff(middle))) < 2 * np.pi * 180.0 / SR * 0.5 * 1.5
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_no_stretch_leaves_the_sound_as_it_is():
|
||||
x = sine(300.0, 0.5)
|
||||
assert np.array_equal(stretch(x, 1.0, SR), x)
|
||||
assert len(stretch(np.zeros(0), 1.5, SR)) == 0
|
||||
|
||||
|
||||
FAKE_CHILD = textwrap.dedent("""
|
||||
import json, struct, sys
|
||||
out = sys.stdout.buffer
|
||||
def reply(h, payload=b""):
|
||||
raw = json.dumps(h).encode()
|
||||
out.write(struct.pack("<I", len(raw)) + raw + payload); out.flush()
|
||||
reply({"ready": True, "rate": 44100})
|
||||
for line in sys.stdin:
|
||||
req = json.loads(line)
|
||||
n = int(req["speed"] * 100)
|
||||
reply({"ok": True, "samples": n}, b"\\x00\\x00" * n)
|
||||
""")
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_engine_sends_its_current_speed_with_every_sentence(tmp_path, monkeypatch):
|
||||
child = tmp_path / "fake_child.py"
|
||||
child.write_text(FAKE_CHILD)
|
||||
monkeypatch.setattr(tts_melo, "CHILD", str(child))
|
||||
engine = tts_melo.MeloEngine(sys.executable, "cpu", 1.5)
|
||||
try:
|
||||
assert len(engine.synthesize("하나", 48000)) == 150
|
||||
engine.speed = 1.8
|
||||
assert len(engine.synthesize("둘", 48000)) == 180
|
||||
finally:
|
||||
engine.close()
|
||||
@@ -43,6 +43,8 @@ class FakeTts:
|
||||
def __init__(self, delay: float = 0.0):
|
||||
self.delay = delay
|
||||
self.texts = []
|
||||
self.speed = 1.5
|
||||
self.speeds = []
|
||||
|
||||
def synthesize(self, text, rate):
|
||||
if self.delay:
|
||||
@@ -50,6 +52,7 @@ class FakeTts:
|
||||
if "FAIL" in text:
|
||||
raise RuntimeError("engine broke")
|
||||
self.texts.append(text)
|
||||
self.speeds.append(self.speed)
|
||||
return np.full(len(text) * 10, 1000, dtype=np.int16)
|
||||
|
||||
|
||||
@@ -272,6 +275,29 @@ def test_a_failing_sentence_ends_the_request_as_failed():
|
||||
assert tts.texts == ["첫 문장입니다."]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_new_speech_speed_applies_to_the_sentences_made_after_it():
|
||||
tts = FakeTts()
|
||||
|
||||
async def body(url):
|
||||
async with connect(url) as ws:
|
||||
await hello(ws)
|
||||
await ws.send(json.dumps({"type": "tts", "id": "a", "text": "보통 속도 문장이에요."}))
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
|
||||
await ws.send(json.dumps({"type": "tts_speed", "speed": 1.8}))
|
||||
out = []
|
||||
for bad in (2.5, 0.2, "fast", True):
|
||||
await ws.send(json.dumps({"type": "tts_speed", "speed": bad}))
|
||||
out.append(json.loads(await ws.recv()))
|
||||
await ws.send(json.dumps({"type": "tts", "id": "b", "text": "빨라진 문장이에요."}))
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
|
||||
return out
|
||||
|
||||
errors = asyncio.run(with_worker(body, tts=tts))
|
||||
assert [e["code"] for e in errors] == ["bad_message"] * 4
|
||||
assert tts.speeds == [1.5, 1.8], "a refused speed leaves the last good one"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_tts_without_an_engine_reports_unavailable():
|
||||
async def body(url):
|
||||
|
||||
Reference in New Issue
Block a user