Compare commits

..

3 Commits

Author SHA1 Message Date
javis-bot
41f539dd48 chore(app): version 0.1.50
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 21:02:05 +09:00
javis-bot
b30aa4a0db feat(voice_worker): a plain decode just short of the gate is rescued by a confident close prompted one
Harsh copies of the user's speech (weak voice, noise at SNR 10/5 dB, headset band, Opus 12 kbps,
reverb) with the worker's own VAD for noise: lowering the gate itself let laughter through (VAD-cut
noise 0/5 -> 1/5 at -0.8), and three noise suppressors (DeepFilterNet3, RNNoise, noisereduce) made
every condition worse, so neither is used. Rescuing a plain decode at -0.8..-0.7 with the prompted
text when that passed the gate and stays close: headset 13.8% -> 7.5%, no more noise with text.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 21:02:05 +09:00
javis-bot
835b547ca2 fix(conversation): a held request survives its answer ending, and is held from the start when the next utterance already began
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 20:37:53 +09:00
10 changed files with 163 additions and 27 deletions

2
app/Cargo.lock generated
View File

@@ -1520,7 +1520,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
[[package]]
name = "jarvis-app"
version = "0.1.49"
version = "0.1.50"
dependencies = [
"async-trait",
"base64 0.22.1",

View File

@@ -1,6 +1,6 @@
[package]
name = "jarvis-app"
version = "0.1.49"
version = "0.1.50"
edition = "2021"
rust-version = "1.85"
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"

4
app/shell/Cargo.lock generated
View File

@@ -2637,7 +2637,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
[[package]]
name = "jarvis-app"
version = "0.1.49"
version = "0.1.50"
dependencies = [
"async-trait",
"base64 0.22.1",
@@ -2666,7 +2666,7 @@ dependencies = [
[[package]]
name = "jarvis-shell"
version = "0.1.49"
version = "0.1.50"
dependencies = [
"jarvis-app",
"serde_json",

View File

@@ -1,6 +1,6 @@
[package]
name = "jarvis-shell"
version = "0.1.49"
version = "0.1.50"
edition = "2021"
rust-version = "1.85"
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"

View File

@@ -361,7 +361,7 @@ impl<V: Voice> Conversation<V> {
let id = format!("q{}", req.id);
if self.voice.ask_addressed(&id, speaker, &text, room) {
self.turns.insert(speaker, Turn { id: req.id, text: req.text.clone(),
ended_at: req.ended_at, ask: Some(id.clone()), held: false, heard: false });
ended_at: req.ended_at, ask: Some(id.clone()), held: self.holds(speaker), heard: false });
self.asks.insert(id, (req, now));
} else {
let a = self.policy.utterance_closed(speaker, now);
@@ -402,7 +402,11 @@ impl<V: Voice> Conversation<V> {
CoreEvent::SpeakEnd(e) => {
if let Some(pid) = self.by_speech.remove(&e.id) {
self.answers.remove(&pid);
self.turns.retain(|_, t| t.id != pid);
// A held request waits for its speaker's next final even after its answer ended.
self.turns.retain(|_, t| t.id != pid || t.held);
for t in self.turns.values_mut().filter(|t| t.id == pid) {
t.heard = true;
}
debug_log!("conversation", "answer req={pid} done played={} cancelled={} failed={}",
e.played, e.cancelled, e.failed);
let a = self.policy.answer_done(pid, now);
@@ -429,10 +433,17 @@ impl<V: Voice> Conversation<V> {
}
}
/// The speaker's next utterance already started (a pause shorter than the first final's
/// transcription): their new request is held open for it from the start.
fn holds(&self, speaker: u64) -> bool {
!self.calls.continue_window.is_zero() && self.started.contains_key(&speaker)
}
/// A voice request goes to the policy; until its answer is heard, its speaker may go on.
fn voice_request(&mut self, req: Request, now: Instant) {
let held = self.holds(req.speaker);
self.turns.insert(req.speaker, Turn { id: req.id, text: req.text.clone(), ended_at: req.ended_at, ask: None,
held: false, heard: false });
held, heard: false });
let a = self.policy.request(req, now);
self.apply(a, now);
}
@@ -454,7 +465,8 @@ impl<V: Voice> Conversation<V> {
return None;
}
let t = self.turns.remove(&speaker).expect("checked");
let earcon_played = self.chimed.remove(&t.id) || (self.calls.earcon && self.answers.contains_key(&t.id));
let earcon_played = self.chimed.remove(&t.id)
|| (self.calls.earcon && (self.answers.contains_key(&t.id) || t.heard));
debug_log!("conversation", "going on req={} speaker={speaker}", t.id);
match t.ask {
Some(q) => {
@@ -1202,6 +1214,74 @@ mod tests {
assert_eq!(f.take(), vec![Did::Cancel("s1".into()), Did::Answer(A, "고마워".into())]);
}
const ALL: [PolicyKind; 4] = [PolicyKind::Latest, PolicyKind::InOrder, PolicyKind::First, PolicyKind::Important];
#[test]
fn going_on_is_joined_even_when_the_short_answer_already_ended() {
// Final -> next SpeechStart -> the answer plays and ends -> next Final.
for kind in ALL {
let (mut c, f, t) = alone(kind);
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
c.on_event(&start(A, 2), ms(t, 1000));
c.on_event(&speak_start("s1"), ms(t, 1500));
c.on_event(&end("s1", false), ms(t, 2500));
c.on_event(&fin_utt(A, 2, "디코 화면 공유 창에 가려져 있잖아"), ms(t, 4000));
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into()),
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())], "{kind:?}");
assert_eq!(c.missed().count(), 0, "{kind:?}");
}
}
#[test]
fn going_on_that_started_before_the_first_final_is_joined() {
// A very short pause: the next utterance starts before the first one's final arrives.
for kind in ALL {
let (mut c, f, t) = alone(kind);
c.on_event(&start(A, 1), ms(t, 0));
c.on_event(&start(A, 2), ms(t, 1000));
c.on_event(&fin_utt(A, 1, "지금 화면 봐봐"), ms(t, 1100));
c.on_event(&speak_start("s1"), ms(t, 1600));
c.on_event(&fin_utt(A, 2, "디코 화면 공유 창에 가려져 있잖아"), ms(t, 3000));
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into()), Did::Cancel("s1".into()),
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())], "{kind:?}");
}
}
#[test]
fn a_held_request_whose_next_final_is_empty_or_a_stop_is_not_asked_again() {
for kind in ALL {
for last in ["", "자비스 멈춰"] {
// (a) the answer already ended.
let (mut c, f, t) = alone(kind);
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
c.on_event(&start(A, 2), ms(t, 1000));
c.on_event(&speak_start("s1"), ms(t, 1500));
c.on_event(&end("s1", false), ms(t, 2500));
c.on_event(&fin_utt(A, 2, last), ms(t, 4000));
let did = f.take();
assert_eq!(did.iter().filter(|d| matches!(d, Did::Answer(..))).count(), 1, "{kind:?} {last:?}");
// What follows is a new request, not joined to the old one.
c.on_event(&fin(A, "고마워", 100), ms(t, 5000));
assert_eq!(f.take().into_iter().filter(|d| matches!(d, Did::Answer(..))).collect::<Vec<_>>(),
vec![Did::Answer(A, "고마워".into())], "{kind:?} {last:?}");
// (b) the next utterance started before the first final, the answer is playing.
let (mut c, f, t) = alone(kind);
c.on_event(&start(A, 1), ms(t, 0));
c.on_event(&start(A, 2), ms(t, 1000));
c.on_event(&fin_utt(A, 1, "지금 화면 봐봐"), ms(t, 1100));
c.on_event(&speak_start("s1"), ms(t, 1600));
c.on_event(&fin_utt(A, 2, last), ms(t, 3000));
let did = f.take();
assert_eq!(did.iter().filter(|d| matches!(d, Did::Answer(..))).count(), 1, "{kind:?} {last:?}");
if last.is_empty() {
assert!(!did.contains(&Did::Cancel("s1".into())), "noise does not stop the answer ({kind:?})");
} else {
assert!(did.contains(&Did::StopAll), "{kind:?}");
}
}
}
}
#[test]
fn going_on_lets_the_brain_get_ready_for_the_joined_request() {
let (mut c, f, t) = alone(PolicyKind::Latest);

View File

@@ -96,8 +96,10 @@ started within `call.continueMs` of the request's end (start = the final's arriv
a stop (a stop stops, as always) and an empty final (noise the worker rejected: nothing is joined
or asked again, the request goes on as it was).
- When the speaker's next utterance starts (`SpeechStart`) within `call.continueMs` of the request's
end, the request is held open for that utterance's final, also if its answer starts playing
meanwhile; the joined request then stops that answer. An empty final ends the hold.
end, or had already started when the request's final came (a pause shorter than the
transcription), the request is held open for that utterance's final, also if its answer starts
playing or even ends meanwhile; the joined request then stops that answer if it still plays. An
empty final or a stop ends the hold, and nothing is asked again.
- Other speakers' utterances never join it. An answer already heard, an answer that ended, a missed
request, a stop, leaving and a lost worker end it; then what the speaker says is a new request.
- The joined request gets no second earcon when the first one's already played.
@@ -253,8 +255,9 @@ off, partials that read as a call sent to get ready (joined to a waiting wake wo
answering someone else, the joined text while going on), going on (pieces joined into one answer in
every policy without a missed record, a piece without a wake word joined, other speakers never, a
new request once the answer is heard or the pause is longer than `call.continueMs`, a stop still
stopping, no second earcon, an utterance started before the answer played still joined, an empty
final joining nothing), interruption cancelling the right speech, a ranking deciding, end times from
stopping, no second earcon, an utterance started before the answer played or before the first final
came still joined, also after a short answer ended, in every policy; an empty final or a stop
after a hold asking nothing again), interruption cancelling the right speech, a ranking deciding, end times from
`wait_ms`, and chat: an answer posted as a reply once, per-channel switches, no answers to bots or to
a stop, chat wake words and designated people, a failed answer posting nothing). `tests/worker_protocol.rs` checks an answer spoken under its id and both judgements against
the real Python worker with a fake brain.

View File

@@ -96,3 +96,8 @@
- 빈 결과의 원인은 확신도 기준(-0.7). -0.85/-1.0으로 낮추면 헤드셋 7.5%, 잡음 10dB 15.5%, 5dB 30.2%, 빈 결과 원본 17→15/10개. 대신 잡음 108개(VAD 없이 바로 넣음)에서 글자가 나온 것이 16→27/32개. 실제 워커는 VAD가 먼저 거르므로(M1: 108개 중 5개만 모델까지 감) 기준 변경은 VAD를 포함한 잡음 시험으로 다시 재고 사용자가 정할 것. 이번에는 바꾸지 않음.
- 작은 목소리+마이크 잡음(약 SNR 5dB)은 어떤 설정으로도 50% 안팎이라, 소리 앞단의 잡음 제거(모델) 같은 다른 방향이 필요함. 아직 미검증.
- 근거: groups/javis_bot/evidence/2026-11-19/stt-plan/harsh/ (build.py 조건 정의, harsh.py, gate.py, 결과 JSON).
- 2026-11-19 밤 (0.1.50): 악조건 두 갈래를 같은 자료(녹음 100개 × 7조건, 정답 16개)로 비교. 근거: evidence/2026-11-19/stt-plan/harsh2/.
- 앞단 잡음 제거 3종(DeepFilterNet3, RNNoise, noisereduce)은 전부 나빠서 넣지 않음. 원본 7.0% → 13.8/9.6/11.0%, 싼 헤드셋 13.8% → 21.8/11.4/19.5%, 잡음 10dB 18.9% → 44.9/48.3/31.6%. DeepFilterNet은 작은 목소리에서 없는 말을 길게 지어냈고(108.6%), VAD를 지난 잡음 5개를 모두 글자로 만듦. 디스코드가 이미 보내는 쪽에서 잡음 제거(Krisp)를 하고, Whisper는 잡음 섞인 소리로 학습되어 제거 흔적이 더 해로움.
- 확신도 기준: 실제 워커처럼 VAD와 분할기로 자른 잡음만 넣으면 모델까지 가는 것은 108개 중 5개(웃음). 기준을 -0.8로만 낮춰도 그중 1개가 "감사합니다."가 되고, VAD 없이 넣은 108개도 16→20개. 그래서 기준 자체는 -0.7 그대로.
- 대신 살리기: 문맥 없는 결과가 -0.7에 조금 못 미쳐도(-0.8 이상) 문맥을 준 결과가 기준을 넘고 가까우면 그것을 씀. 실제 코드로 다시 잼(verify.py): 원본 7.0%, 헤드셋 13.8→7.5%, 빈 결과 대부분 1~2개 줄어듦, 잡음 0/5와 16/108 그대로, 최종 전사 평균 178~180ms(p95 246~254ms), 10명 한 묶음 1.1초.
- 남은 문제(수치 그대로): 작은 목소리+마이크 잡음 51.4%(빈 결과 48/100), 잡음 5dB 33.9%(34/100), 잡음 10dB 18.9%(23/100). 이번에 본 방법(모델 7종, 문맥, 기준, 잡음 제거 3종)으로는 나아지지 않음. 다음 후보는 이런 조건의 실제 녹음으로 미세조정한 모델이나, 보내는 사람 쪽 디스코드 설정(입력 감도, 잡음 제거) 안내.

View File

@@ -19,9 +19,13 @@ SR = 16000
# over it: every hallucination stayed below, real speech above except two short words in loud noise.
# Whisper's no_speech_prob is not used: large-v3-turbo reports 0.0 for silence and noise alike.
MIN_MEAN_LOGPROB = -0.7
MAX_COMPRESSION_RATIO = 2.4
MAX_COMPRESSION_RATIO = 2.4 # repetition loops ("A B A B A B ...") compress far better than speech
# faster-whisper keeps at most this many previous-text tokens in a prompt (max_length // 2 - 1).
MAX_PREVIOUS_TOKENS = 223 # repetition loops ("A B A B A B ...") compress far better than speech
MAX_PREVIOUS_TOKENS = 223
# A plain decode at or above this, though below MIN_MEAN_LOGPROB, may be rescued by the context
# (choose_with_context). Measured 2026-11-19 on harsh copies of the user's speech: down to -0.8 no
# more noise came out as text (VAD-cut noise 0/5, raw noise 16/108 as without); -0.9 let 3 more through.
UNSURE_LOGPROB = -0.8
def _letters(text: str) -> str:
@@ -75,16 +79,24 @@ def choose_with_hint(hint_text: str, hint_lp: float, plain_text: str, plain_lp:
return plain_text
def choose_with_context(heard: str, prompted: str, hotwords: tuple[str, ...]) -> str:
def choose_with_context(heard: str, prompted: str, hotwords: tuple[str, ...],
plain: tuple[str, float] | None = None) -> str:
"""Pick between the decode without the conversation context (``heard``, already through the hint
choice) and the one that had it as Whisper's previous-text prompt (``prompted``).
choice) and the one that had it as Whisper's previous-text prompt (``prompted``, through the same
choice, so it passed the gate itself).
The context fixes words of the topic the model misheard, but on noise or unsure audio it can write
words of the prompt, so the decode without it decides, as for the hint: nothing heard stays
nothing, and the prompted text is used only when it stays close to what was heard and holds every
hotword exactly as often.
words of the prompt, so the decode without it decides, as for the hint: the prompted text is used
only when it stays close to what was heard and holds every hotword exactly as often. When nothing
was heard, ``plain`` (the plain decode's text and mean log-probability) may still be just short of
the gate (at least UNSURE_LOGPROB): then the prompted text is used if it is that close to it.
"""
if not heard or not prompted:
if not heard:
if not prompted or plain is None or plain[1] < UNSURE_LOGPROB \
or compression_ratio(plain[0]) > MAX_COMPRESSION_RATIO:
return ""
return prompted if choose_with_context(plain[0], prompted, hotwords) == prompted else ""
if not prompted:
return heard
h, p = _letters(heard), _letters(prompted)
if _edit_distance(p, h) > len(h) // 2:
@@ -130,13 +142,14 @@ class WhisperTranscriber:
"""Clips arrive loudness-equalised by the session. ``contexts``: per clip, the recent
conversation to prompt with ("" for none); see ``choose_with_context``."""
enc, langs = self._encode(clips, opts)
heard = self._choose(enc, langs, opts, None)
plain = self._decode(enc, langs, opts.beam, None)
heard = self._choose(enc, langs, opts, None, plain)
prev = self._previous(langs, contexts)
if prev is None:
return heard
prompted = self._choose(enc, langs, opts, prev)
return [choose_with_context(h, p, opts.hotwords) if c else h
for h, p, c in zip(heard, prompted, contexts)]
prompted = self._choose(enc, langs, opts, prev, self._decode(enc, langs, opts.beam, None, prev))
return [choose_with_context(h, p, opts.hotwords, pl) if c else h
for h, p, pl, c in zip(heard, prompted, plain, contexts)]
def _previous(self, langs: list[str], contexts: list[str] | None) -> list[list[int]] | None:
"""The contexts as previous-text tokens of one length for the whole batch: the decoder needs
@@ -166,8 +179,8 @@ class WhisperTranscriber:
langs = [r[0][0][2:-2] for r in m.model.detect_language(enc)]
return enc, langs
def _choose(self, enc, langs: list[str], opts: DecodeOptions, prev: list[list[int]] | None) -> list[str]:
plain = self._decode(enc, langs, opts.beam, None, prev)
def _choose(self, enc, langs: list[str], opts: DecodeOptions, prev: list[list[int]] | None,
plain: list[tuple[str, float]]) -> list[str]:
if not opts.hotwords:
return [text if keep_transcript(text, lp) else "" for text, lp in plain]
hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords), prev)

View File

@@ -171,6 +171,12 @@ decoded with the recent conversation as its previous-text prompt:
was kept, it stays within half that text's letters of it, and it holds every hotword exactly as
often; otherwise the text without context is used. On noise or unsure audio the prompt makes
Whisper write words of the conversation; this rule keeps that out, as for the hint.
- One rescue: when the decode without context was dropped by the gate but its mean log-probability
is at least -0.8 (just short of -0.7), the prompted text (which passed the gate itself) is used if
it is that close to it and keeps every hotword count. The unsure text itself is never used.
Measured on harsh copies of the user's speech: a cheap headset 13.8% to 7.5% error, empty results
fewer by 1-2 per 100 in most conditions, and no more noise with text (VAD-cut noise 0 of 5, noise
decoded straight 16 of 108, as before); -0.9 let 3 more noise clips through.
- Finals stay batched. The decoder needs the start token at the same position in every prompt of a
batch, so each clip's context is cut from its end to the shortest one in the batch (a channel's
clips share most of it); a clip without a context borrows another's and keeps its plain result.

View File

@@ -132,3 +132,32 @@ def test_the_context_never_changes_how_many_wake_words_were_heard():
@pytest.mark.unit
def test_an_empty_prompted_decode_keeps_what_was_heard():
assert choose_with_context("네.", "", HINT) == "네."
# A plain decode just short of the gate (live 2026-11-19 on a headset copy: "니코 어디갔어" at -0.72 for
# "디코 어디 갔어") is rescued by a confident prompted decode close to it; nothing else changes.
from voice_worker.stt_whisper import UNSURE_LOGPROB # noqa: E402
@pytest.mark.unit
def test_a_nearly_sure_decode_is_rescued_by_a_confident_close_prompted_one():
assert choose_with_context("", "디코 어디 갔어?", HINT, ("니코 어디갔어", MIN_MEAN_LOGPROB - 0.02)) \
== "디코 어디 갔어?"
@pytest.mark.unit
def test_an_unsure_decode_well_below_the_gate_is_not_rescued():
assert choose_with_context("", "디코 어디 갔어?", HINT, ("니코 어디갔어", UNSURE_LOGPROB - 0.05)) == ""
@pytest.mark.unit
def test_noise_with_a_prompt_far_from_it_is_not_rescued():
# The prompted decode wrote words of the conversation over laughter.
assert choose_with_context("", "화면에 디스코드", HINT, ("감사합니다.", MIN_MEAN_LOGPROB - 0.05)) == ""
assert choose_with_context("", "", HINT, ("감사합니다.", MIN_MEAN_LOGPROB - 0.05)) == ""
@pytest.mark.unit
def test_a_rescue_never_adds_a_wake_word():
assert choose_with_context("", "자비스 지금 영상 재생되고 있어?", HINT,
("라비스 지금 영상 재생되고 있어", MIN_MEAN_LOGPROB - 0.05)) == ""