From e716065538ca5c3c13ec4fa9a69f388040b51e17 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Thu, 19 Nov 2026 21:55:10 +0900 Subject: [PATCH] fix(voice_worker): a hinted wake word may replace a near miss the plain decode wrote MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live 2026-11-19 "비서야" was decoded as "기사야" (two of six letters off, more than call detection allows) while the hinted decode of the same audio gave "비서야"; the hint was dropped because it changed the wake word count. The hint may now add one wake word where the plain text is within a third of that word's letters (NFKD letters, Hangul by jamo). On the user's 100 recordings: 5 fixed, 0 wrong; 0 of 216 noise decodes gained a wake word. The live round trip test gets a going-on case: the person's own recordings of one thing said in pieces, played into Discord with their real pauses. Co-Authored-By: Claude Opus 5.5 --- app/tests/live_round_trip.rs | 138 +++++++++++++++++++++++--- docs/v2/stt-accuracy-plan.md | 3 + src/voice_worker/stt_whisper.py | 41 +++++++- src/voice_worker/voice_worker.spec.md | 21 +++- tests/test_voice_worker_gate.py | 27 +++-- 5 files changed, 197 insertions(+), 33 deletions(-) diff --git a/app/tests/live_round_trip.rs b/app/tests/live_round_trip.rs index 0bf12be..c3c1666 100644 --- a/app/tests/live_round_trip.rs +++ b/app/tests/live_round_trip.rs @@ -17,6 +17,7 @@ //! cargo test --release --test live_round_trip -- --nocapture //! //! On Linux, put the pip CUDA libraries on LD_LIBRARY_PATH (nvidia/cublas/lib, nvidia/cudnn/lib). +//! JARVIS_LIVE_WORKER_ARGS adds worker arguments (e.g. "--stt-device cpu --stt-compute int8"). use std::collections::HashMap; use std::path::{Path, PathBuf}; @@ -165,16 +166,23 @@ fn background_voices(url: String, token: String, samples: Vec>, stop: A }) } -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn conversation_round_trip_on_real_discord() { - if std::env::var("JARVIS_LIVE_ROUND_TRIP").as_deref() != Ok("1") { - eprintln!("skipped: set JARVIS_LIVE_ROUND_TRIP=1"); - return; - } +/// Both bots in the voice channel: the speaker bot (the "person") and Jarvis running the app's services +/// from a fresh data folder, with `call` as the settings' call section. +struct Live { + speaker_id: u64, + speaker_songbird: Arc, + s_shards: Arc, + services: jarvis_app::services::Services, + core: jarvis_app::discord::DiscordCore, + events: Arc>>, + call: Arc>, + ears: Ears, + guild: u64, +} + +async fn start_live(call_settings: impl FnOnce(u64) -> Value) -> Live { let guild: u64 = env("JARVIS_LIVE_GUILD").parse().unwrap(); let voice: u64 = env("JARVIS_LIVE_VOICE_CHANNEL").parse().unwrap(); - let rounds: usize = std::env::var("JARVIS_LIVE_ROUNDS").ok().and_then(|v| v.parse().ok()).unwrap_or(2); - let questions = PathBuf::from(env("JARVIS_LIVE_QUESTIONS")); env("JARVIS_LIVE_LLM_URL"); // the services point the brain at it (llm.url) let root = Path::new(env!("CARGO_MANIFEST_DIR")); @@ -208,21 +216,26 @@ async fn conversation_round_trip_on_real_discord() { if let Ok(m) = std::env::var("JARVIS_MELO_PYTHON") { args.extend(["--melo-python".to_string(), m]); } + // e.g. "--stt-device cpu --stt-compute int8 --melo-device cpu" on a host without a GPU + if let Ok(extra) = std::env::var("JARVIS_LIVE_WORKER_ARGS") { + args.extend(extra.split_whitespace().map(String::from)); + } std::fs::write(data.join("config").join("settings.json"), serde_json::to_string_pretty(&json!({ "discord": {"guildId": guild.to_string(), "voiceChannelId": voice.to_string()}, // The "person" here is a bot, so bots must be heard; the recorded questions call "자비스". "voice": {"listenToBots": true, "wakeWords": ["자비스"]}, + "call": call_settings(speaker_id), "voiceWorker": {"python": env("JARVIS_VOICE_PYTHON"), "cwd": root.join("../src"), "args": args}, "llm": {"url": env("JARVIS_LIVE_LLM_URL")}, })).unwrap()).unwrap(); std::env::set_var("JARVIS_DISCORD_TOKEN", &bot_token); - let services = tokio::time::timeout(Duration::from_secs(300), start_services(&data, Arc::new(Llm::new(&data)))).await + let mut services = tokio::time::timeout(Duration::from_secs(300), start_services(&data, Arc::new(Llm::new(&data)))).await .expect("services start in time").expect("services start"); services.worker.wait_ready(Duration::from_secs(240)).await.expect("worker ready"); let core = services.core.clone(); let events: Arc>> = Arc::default(); let ev2 = events.clone(); - let mut rx = services.events; + let mut rx = std::mem::replace(&mut services.events, mpsc::unbounded_channel().1); tokio::spawn(async move { while let Some(e) = rx.recv().await { ev2.lock().unwrap().push((Instant::now(), e)); @@ -240,6 +253,31 @@ async fn conversation_round_trip_on_real_discord() { } tokio::time::sleep(Duration::from_secs(2)).await; // the DAVE group settles after the join + Live { speaker_id, speaker_songbird, s_shards, services, core, events, call, ears, guild } +} + +impl Live { + async fn stop(self) { + let _ = self.speaker_songbird.remove(GuildId::new(self.guild)).await; + self.s_shards.shutdown_all().await; + self.services.core.leave("test").await; + self.services.core.close().await; + self.services.worker.close(); + self.services.worker_process.stop().await; + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn conversation_round_trip_on_real_discord() { + if std::env::var("JARVIS_LIVE_ROUND_TRIP").as_deref() != Ok("1") { + eprintln!("skipped: set JARVIS_LIVE_ROUND_TRIP=1"); + return; + } + let live = start_live(|_| json!({})).await; + let rounds: usize = std::env::var("JARVIS_LIVE_ROUNDS").ok().and_then(|v| v.parse().ok()).unwrap_or(2); + let questions = PathBuf::from(env("JARVIS_LIVE_QUESTIONS")); + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let Live { speaker_id, ref services, ref core, ref events, ref call, ref ears, .. } = live; let refs: Value = serde_json::from_str(&std::fs::read_to_string(questions.join("refs.json")).unwrap()).unwrap(); let qs: Vec<(String, Vec)> = refs["samples"].as_array().unwrap().iter() .map(|r| (r["text"].as_str().unwrap().to_owned(), std::fs::read(questions.join(r["file"].as_str().unwrap())).unwrap())) @@ -355,12 +393,7 @@ async fn conversation_round_trip_on_real_discord() { println!("live round trip {}", serde_json::to_string_pretty(&Value::Object(report.clone())).unwrap()); })).await; - let _ = speaker_songbird.remove(GuildId::new(guild)).await; - s_shards.shutdown_all().await; - services.core.leave("test").await; - services.core.close().await; - services.worker.close(); - services.worker_process.stop().await; + live.stop().await; if let Err(panic) = steps { std::panic::resume_unwind(panic); } @@ -371,3 +404,76 @@ async fn conversation_round_trip_on_real_discord() { } } + +/// Going on (conversation.spec.md), live: the person's own recordings of one thing said in pieces, +/// played with the pauses they really made, get one answer. Settings as the person's own app: they +/// are the only designated person, `addressed` mode, alone with Jarvis. +/// +/// JARVIS_LIVE_GOING_ON=1 (the env of conversation_round_trip_on_real_discord, without the questions) +/// JARVIS_LIVE_PIECES= cargo test --release --test live_round_trip going_on -- --nocapture +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn going_on_on_real_discord() { + if std::env::var("JARVIS_LIVE_GOING_ON").as_deref() != Ok("1") { + eprintln!("skipped: set JARVIS_LIVE_GOING_ON=1"); + return; + } + let dir = PathBuf::from(env("JARVIS_LIVE_PIECES")); + let seqs: Value = serde_json::from_str(&std::fs::read_to_string(dir.join("pieces.json")).unwrap()).unwrap(); + let live = start_live(|speaker| json!({"designated": [speaker.to_string()], "designatedOnly": true, "mode": "addressed"})).await; + let mut rows = vec![]; + let steps = futures_util::FutureExt::catch_unwind(std::panic::AssertUnwindSafe(async { + for s in seqs.as_array().unwrap() { + let wav = std::fs::read(dir.join(s["file"].as_str().unwrap())).unwrap(); + until("Jarvis to finish speaking", 60_000, || !live.core.speaking()).await; + let before = live.events.lock().unwrap().len(); + live.call.lock().await.play_only_input(Input::from(wav.clone())); + tokio::time::sleep(Duration::from_secs_f64(speech_end_seconds(&wav))).await; + // Done when nothing is playing and nothing new came for 10 s (at most 60 s). + let deadline = Instant::now() + Duration::from_secs(60); + let (mut seen, mut quiet_since) = (0, Instant::now()); + while Instant::now() < deadline { + let n = live.events.lock().unwrap().len(); + if n != seen || live.core.speaking() { + (seen, quiet_since) = (n, Instant::now()); + } else if quiet_since.elapsed() > Duration::from_secs(10) { + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + let ev = live.events.lock().unwrap()[before..].to_vec(); + let heard: Vec = ev.iter().filter_map(|(_, e)| match e { + CoreEvent::Final(f) if f.heard.speaker_id == live.speaker_id => Some(f.text.clone()), + _ => None, + }).collect(); + let answers: Vec = ev.iter().filter_map(|(_, e)| match e { + CoreEvent::ReplyText { id, text, failed } => { + let end = ev.iter().find_map(|(_, e)| match e { + CoreEvent::SpeakEnd(x) if &x.id == id => Some(x.clone()), + _ => None, + }); + Some(json!({"text": text, "failed": failed, "played": end.as_ref().map(|x| x.played), + "cancelled": end.as_ref().map(|x| x.cancelled)})) + } + _ => None, + }).collect(); + let asked = ev.iter().filter(|(_, e)| matches!(e, CoreEvent::AnswerAsked { .. })).count(); + rows.push(json!({"name": s["name"], "heard": heard, "asked": asked, "answers": answers})); + println!("going on {}", rows.last().unwrap()); + tokio::time::sleep(Duration::from_millis(1500)).await; + } + })).await; + live.stop().await; + if let Err(panic) = steps { + std::panic::resume_unwind(panic); + } + // One thing said in pieces: one answer heard to its end, and no other answer heard. + for r in &rows { + let heard = |a: &&Value| a["played"].as_u64().is_some_and(|p| p > 0); + let whole: Vec<&Value> = r["answers"].as_array().unwrap().iter().filter(heard) + .filter(|a| a["cancelled"] == json!(false)).collect(); + let cut: Vec<&Value> = r["answers"].as_array().unwrap().iter().filter(heard) + .filter(|a| a["cancelled"] == json!(true)).collect(); + assert!(whole.len() == 1 && cut.is_empty(), "{}: not one answer: {r}", r["name"]); + } +} diff --git a/docs/v2/stt-accuracy-plan.md b/docs/v2/stt-accuracy-plan.md index 31a45a3..9a54b7f 100644 --- a/docs/v2/stt-accuracy-plan.md +++ b/docs/v2/stt-accuracy-plan.md @@ -101,3 +101,6 @@ - 확신도 기준: 실제 워커처럼 VAD와 분할기로 자른 잡음만 넣으면 모델까지 가는 것은 108개 중 5개(웃음). 기준을 -0.8로만 낮춰도 그중 1개가 "감사합니다."가 되고, VAD 없이 넣은 108개도 16→20개. 그래서 기준 자체는 -0.7 그대로. - 대신 살리기: 문맥 없는 결과가 -0.7에 조금 못 미쳐도(-0.8 이상) 문맥을 준 결과가 기준을 넘고 가까우면 그것을 씀. 실제 코드로 다시 잼(verify.py): 원본 7.0%, 헤드셋 13.8→7.5%, 빈 결과 대부분 1~2개 줄어듦, 잡음 0/5와 16/108 그대로, 최종 전사 평균 178~180ms(p95 246~254ms), 10명 한 묶음 1.1초. - 남은 문제(수치 그대로): 작은 목소리+마이크 잡음 51.4%(빈 결과 48/100), 잡음 5dB 33.9%(34/100), 잡음 10dB 18.9%(23/100). 이번에 본 방법(모델 7종, 문맥, 기준, 잡음 제거 3종)으로는 나아지지 않음. 다음 후보는 이런 조건의 실제 녹음으로 미세조정한 모델이나, 보내는 사람 쪽 디스코드 설정(입력 감도, 잡음 제거) 안내. +- 2026-11-19 밤 (0.1.50 뒤): 근거 evidence/2026-11-19/stt-plan/wake/. + - 첫마디 호출어: 힌트를 준 결과는 "비서야"를 맞게 들었는데(기사야·기서야·라비스·아비스 → 비서야·자비스), 힌트 결과는 힌트 없는 결과와 호출어 수가 같아야만 써서 버려졌음. 규칙을 바꿈: 힌트 없는 결과에 그 호출어의 비슷한 말이 있으면(글 전체가 호출어 글자 수의 3분의 1 이내, 한글은 자모로 셈) 호출어 하나를 더할 수 있음. 녹음 100개에서 5개를 바로잡고 3개는 그대로(그중 "기구"→"김비서"는 실제로 "디코"라 막은 것이 맞음), 잡음 216번(108개 × 힌트 2종)에서 더한 호출어 0. 소리로 다시 매기기(align 확률)는 맞는 쪽과 틀린 쪽을 가르지 못해 쓰지 않음. + - 실제 디스코드(테스트 봇이 사용자 녹음을 실제 쉰 간격대로 틀음, app/tests/live_round_trip.rs going_on, 서버 CPU, LLM은 질문을 되돌려 주는 대역): 부산 날씨 3조각은 한 질문, 한 답. 계산기 6조각(쉼 2.3~4.7초)은 쉼이 답 만드는 시간보다 길어서 답이 먼저 들리기 시작했고, 들린 뒤의 말은 새 질문이 되어 중간 답 2개가 끊김. 서버 CPU라 받아 적기가 10~25초 걸려 시간은 실제와 다름(정확한 시간은 게임PC GPU에서 다시 재야 함). diff --git a/src/voice_worker/stt_whisper.py b/src/voice_worker/stt_whisper.py index 2e5052f..79f2689 100644 --- a/src/voice_worker/stt_whisper.py +++ b/src/voice_worker/stt_whisper.py @@ -6,6 +6,7 @@ import glob import os import re import sys +import unicodedata import zlib import numpy as np @@ -61,20 +62,50 @@ def _mentions(text: str, word: str) -> int: return _letters(text).count(w) if w else 0 +def _calls(text: str, hotwords: tuple[str, ...]) -> list[str]: + """The wake words in ``text``, longest first at each place, without overlaps ("김비서야" is one).""" + s = _letters(text) + words = sorted({_letters(w) for w in hotwords if _letters(w)}, key=len, reverse=True) + found, i = [], 0 + while i < len(s): + w = next((w for w in words if s.startswith(w, i)), None) + if w: + found.append(w) + i += len(w) + else: + i += 1 + return found + + +def keeps_wake_words(text: str, base: str, hotwords: tuple[str, ...]) -> bool: + """Whether ``text`` (a hinted or prompted decode) holds the wake words of ``base`` (the decode it is + checked against): every hotword exactly as often, or one wake word more where ``base`` has a near + miss of it. Near: the whole text is at most a third of that wake word's letters from ``base``, + compared as Unicode NFKD letters (Hangul by its letters: "기사야" is two of six off "비서야").""" + if all(_mentions(text, w) == _mentions(base, w) for w in hotwords): + return True + have, had = _calls(text, hotwords), _calls(base, hotwords) + if len(have) != len(had) + 1 or any(_mentions(text, w) < _mentions(base, w) for w in hotwords): + return False + nfkd = lambda t: unicodedata.normalize("NFKD", _letters(t)) + return _edit_distance(nfkd(text), nfkd(base)) <= min(len(nfkd(w)) for w in have) // 3 + + def choose_with_hint(hint_text: str, hint_lp: float, plain_text: str, plain_lp: float, hotwords: tuple[str, ...]) -> str: """Pick between the decode that had the hotwords as a hint and the plain decode of the same audio. The hint can make Whisper write a wake word that nobody said (on laughter, tacked onto a real - sentence, or in place of a similar word), so the plain decode decides: no plain text means nothing - was said, and the hint's text is used only when it stays close to the plain one and contains every - hotword exactly as often. It may change other words, never how many wake words were heard. + sentence, or in place of another word), so the plain decode decides: no plain text means nothing + was said, and the hint's text is used only when it stays close to the plain one and keeps its wake + words (``keeps_wake_words``): it may change other words, and turn a near miss of a wake word into it. """ if not keep_transcript(plain_text, plain_lp): return "" close = _edit_distance(_letters(hint_text), _letters(plain_text)) <= len(_letters(plain_text)) // 2 - if close and keep_transcript(hint_text, hint_lp) and all( - _mentions(hint_text, w) == _mentions(plain_text, w) for w in hotwords): + if close and keep_transcript(hint_text, hint_lp) and keeps_wake_words(hint_text, plain_text, hotwords): + if len(_calls(hint_text, hotwords)) != len(_calls(plain_text, hotwords)): + debug_log("hint turned a near miss into a wake word", "stt") return hint_text return plain_text diff --git a/src/voice_worker/voice_worker.spec.md b/src/voice_worker/voice_worker.spec.md index 894d785..9db2910 100644 --- a/src/voice_worker/voice_worker.spec.md +++ b/src/voice_worker/voice_worker.spec.md @@ -145,11 +145,22 @@ text is empty: - Its compression ratio (UTF-8 bytes over zlib bytes) is at most 2.4, which catches repetition loops. - With hotwords: the clip is decoded once with them and once without, from the same encoder pass. The plain decode must pass the checks above, otherwise nothing was said. The hinted text is used - only when it stays within half the plain text's letters of it and contains every hotword exactly - as often as the plain one (ignoring case, spacing and punctuation, no fuzzy matching), so the hint - can never add, drop or create a wake word: not from laughter, not after a real sentence, not from - a one-letter wake word, and not by turning a similar word ("서비스", or the near miss "자비수") - into it. Otherwise the plain text is used; matching near misses is left to call detection. + only when it stays within half the plain text's letters of it and keeps the plain one's wake + words (ignoring case, spacing and punctuation): every hotword exactly as often, or exactly one + wake word more where the plain text has a near miss of it, that is when the whole hinted text is + at most a third of that wake word's letters from the plain one, compared as Unicode NFKD letters + (Hangul by its letters). So the hint never drops a wake word, and never adds one from laughter, + after a real sentence, from a one-letter wake word or in place of a different word ("기구" heard + for "디코" became "김비서" with the hint: five letters off). It does turn a near miss the plain + decode wrote into the wake word the hinted decode heard: live 2026-11-19 "비서야" was decoded as + "기사야" (two of six letters off, more than call detection allows), and the hinted decode of the + same audio gave "비서야". Otherwise the plain text is used; other near misses are left to call + detection. Measured on the user's 100 recordings and 108 noise clips (CPU int8, 2026-11-19): the + hint added a wake word to 8 recordings and 15 noise clips; this rule takes 5 (all said: 자비스 or + 비서야 heard as 라비스, 아비스, 기사야, 기서야) and rejects the 3 others (two said, one "디코") and all + 15 noise clips. Of 15 synthetic sentences with words near the wake words, the hint added none + where the word was heard as said (서비스, 기사, 이사야, 비상약, 미소야, 시사); two where the voice + itself said "비사(야)" for "기사(야)" became "비서(야)". `no_speech_prob` is not used: large-v3-turbo reports 0.0 for silence and noise alike. diff --git a/tests/test_voice_worker_gate.py b/tests/test_voice_worker_gate.py index 27833fb..3d27747 100644 --- a/tests/test_voice_worker_gate.py +++ b/tests/test_voice_worker_gate.py @@ -56,16 +56,29 @@ def test_a_one_letter_wake_word_the_hint_adds_is_dropped(): @pytest.mark.unit -def test_the_hint_may_not_turn_a_similar_real_word_into_the_wake_word(): - assert choose_with_hint("자비스 신청해 줘", -0.1, "서비스 신청해 줘", -0.1, HINT) == "서비스 신청해 줘" +def test_the_hint_may_not_turn_a_different_word_into_the_wake_word(): + # Live 2026-11-19: "디코 어디 갔어?" decoded as "기구 어디 갔어?", and with the hint as "김비서 어디 갔어?". + secretary = ("비서", "비서야", "김비서") + assert choose_with_hint("김비서 어디 갔어?", -0.49, "기구 어디 갔어?", -0.59, secretary) == "기구 어디 갔어?" @pytest.mark.unit -def test_a_near_miss_spelling_is_left_as_heard_rather_than_guessed(): - # "자비수" may be the wake word or another word; text alone cannot tell, so the plain decode stands - # and fuzzy wake word matching is left to the call detection that reads it. - plain = "자비수, 오늘 서울 날씨 어때?" - assert choose_with_hint("자비스, 오늘 서울 날씨 어때?", -0.1, plain, -0.2, HINT) == plain +def test_a_hinted_wake_word_replaces_a_near_miss_of_it(): + # Live 2026-11-19: "비서야 디코 화면공유 보여?" decoded as "기사야 ...", two letters off "비서야" + # (more than call detection allows); decoded with the hint, the same audio gave "비서야". + secretary = ("비서", "비서야", "김비서") + hinted = "비서야 디코 화면군이 보여" + assert choose_with_hint(hinted, -0.19, "기사야 디코 화면군이 보여.", -0.25, secretary) == hinted + hinted = "자비스, 오늘 서울 날씨 어때?" + assert choose_with_hint(hinted, -0.1, "자비수, 오늘 서울 날씨 어때?", -0.2, HINT) == hinted + + +@pytest.mark.unit +def test_a_near_miss_is_replaced_only_by_one_wake_word(): + assert choose_with_hint("자비스 자비스 날씨", -0.1, "자비수 자비수 날씨", -0.1, HINT) == "자비수 자비수 날씨" + # Nor when the hint also changes much else: the budget is the whole text's difference. + assert choose_with_hint("자비스 오늘 날씨 알려 줘", -0.1, "자비수 어제 날씨 알려 줘", -0.1, HINT) \ + == "자비수 어제 날씨 알려 줘" @pytest.mark.unit