Compare commits
3 Commits
app-v0.1.4
...
app-v0.1.5
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
41f539dd48 | ||
|
|
b30aa4a0db | ||
|
|
835b547ca2 |
2
app/Cargo.lock
generated
2
app/Cargo.lock
generated
@@ -1520,7 +1520,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.49"
|
||||
version = "0.1.50"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.49"
|
||||
version = "0.1.50"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"
|
||||
|
||||
4
app/shell/Cargo.lock
generated
4
app/shell/Cargo.lock
generated
@@ -2637,7 +2637,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.49"
|
||||
version = "0.1.50"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -2666,7 +2666,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.49"
|
||||
version = "0.1.50"
|
||||
dependencies = [
|
||||
"jarvis-app",
|
||||
"serde_json",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.49"
|
||||
version = "0.1.50"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"
|
||||
|
||||
@@ -361,7 +361,7 @@ impl<V: Voice> Conversation<V> {
|
||||
let id = format!("q{}", req.id);
|
||||
if self.voice.ask_addressed(&id, speaker, &text, room) {
|
||||
self.turns.insert(speaker, Turn { id: req.id, text: req.text.clone(),
|
||||
ended_at: req.ended_at, ask: Some(id.clone()), held: false, heard: false });
|
||||
ended_at: req.ended_at, ask: Some(id.clone()), held: self.holds(speaker), heard: false });
|
||||
self.asks.insert(id, (req, now));
|
||||
} else {
|
||||
let a = self.policy.utterance_closed(speaker, now);
|
||||
@@ -402,7 +402,11 @@ impl<V: Voice> Conversation<V> {
|
||||
CoreEvent::SpeakEnd(e) => {
|
||||
if let Some(pid) = self.by_speech.remove(&e.id) {
|
||||
self.answers.remove(&pid);
|
||||
self.turns.retain(|_, t| t.id != pid);
|
||||
// A held request waits for its speaker's next final even after its answer ended.
|
||||
self.turns.retain(|_, t| t.id != pid || t.held);
|
||||
for t in self.turns.values_mut().filter(|t| t.id == pid) {
|
||||
t.heard = true;
|
||||
}
|
||||
debug_log!("conversation", "answer req={pid} done played={} cancelled={} failed={}",
|
||||
e.played, e.cancelled, e.failed);
|
||||
let a = self.policy.answer_done(pid, now);
|
||||
@@ -429,10 +433,17 @@ impl<V: Voice> Conversation<V> {
|
||||
}
|
||||
}
|
||||
|
||||
/// The speaker's next utterance already started (a pause shorter than the first final's
|
||||
/// transcription): their new request is held open for it from the start.
|
||||
fn holds(&self, speaker: u64) -> bool {
|
||||
!self.calls.continue_window.is_zero() && self.started.contains_key(&speaker)
|
||||
}
|
||||
|
||||
/// A voice request goes to the policy; until its answer is heard, its speaker may go on.
|
||||
fn voice_request(&mut self, req: Request, now: Instant) {
|
||||
let held = self.holds(req.speaker);
|
||||
self.turns.insert(req.speaker, Turn { id: req.id, text: req.text.clone(), ended_at: req.ended_at, ask: None,
|
||||
held: false, heard: false });
|
||||
held, heard: false });
|
||||
let a = self.policy.request(req, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
@@ -454,7 +465,8 @@ impl<V: Voice> Conversation<V> {
|
||||
return None;
|
||||
}
|
||||
let t = self.turns.remove(&speaker).expect("checked");
|
||||
let earcon_played = self.chimed.remove(&t.id) || (self.calls.earcon && self.answers.contains_key(&t.id));
|
||||
let earcon_played = self.chimed.remove(&t.id)
|
||||
|| (self.calls.earcon && (self.answers.contains_key(&t.id) || t.heard));
|
||||
debug_log!("conversation", "going on req={} speaker={speaker}", t.id);
|
||||
match t.ask {
|
||||
Some(q) => {
|
||||
@@ -1202,6 +1214,74 @@ mod tests {
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s1".into()), Did::Answer(A, "고마워".into())]);
|
||||
}
|
||||
|
||||
const ALL: [PolicyKind; 4] = [PolicyKind::Latest, PolicyKind::InOrder, PolicyKind::First, PolicyKind::Important];
|
||||
|
||||
#[test]
|
||||
fn going_on_is_joined_even_when_the_short_answer_already_ended() {
|
||||
// Final -> next SpeechStart -> the answer plays and ends -> next Final.
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1500));
|
||||
c.on_event(&end("s1", false), ms(t, 2500));
|
||||
c.on_event(&fin_utt(A, 2, "디코 화면 공유 창에 가려져 있잖아"), ms(t, 4000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into()),
|
||||
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())], "{kind:?}");
|
||||
assert_eq!(c.missed().count(), 0, "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_that_started_before_the_first_final_is_joined() {
|
||||
// A very short pause: the next utterance starts before the first one's final arrives.
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&start(A, 1), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&fin_utt(A, 1, "지금 화면 봐봐"), ms(t, 1100));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1600));
|
||||
c.on_event(&fin_utt(A, 2, "디코 화면 공유 창에 가려져 있잖아"), ms(t, 3000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into()), Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())], "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_request_whose_next_final_is_empty_or_a_stop_is_not_asked_again() {
|
||||
for kind in ALL {
|
||||
for last in ["", "자비스 멈춰"] {
|
||||
// (a) the answer already ended.
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1500));
|
||||
c.on_event(&end("s1", false), ms(t, 2500));
|
||||
c.on_event(&fin_utt(A, 2, last), ms(t, 4000));
|
||||
let did = f.take();
|
||||
assert_eq!(did.iter().filter(|d| matches!(d, Did::Answer(..))).count(), 1, "{kind:?} {last:?}");
|
||||
// What follows is a new request, not joined to the old one.
|
||||
c.on_event(&fin(A, "고마워", 100), ms(t, 5000));
|
||||
assert_eq!(f.take().into_iter().filter(|d| matches!(d, Did::Answer(..))).collect::<Vec<_>>(),
|
||||
vec![Did::Answer(A, "고마워".into())], "{kind:?} {last:?}");
|
||||
// (b) the next utterance started before the first final, the answer is playing.
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&start(A, 1), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&fin_utt(A, 1, "지금 화면 봐봐"), ms(t, 1100));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1600));
|
||||
c.on_event(&fin_utt(A, 2, last), ms(t, 3000));
|
||||
let did = f.take();
|
||||
assert_eq!(did.iter().filter(|d| matches!(d, Did::Answer(..))).count(), 1, "{kind:?} {last:?}");
|
||||
if last.is_empty() {
|
||||
assert!(!did.contains(&Did::Cancel("s1".into())), "noise does not stop the answer ({kind:?})");
|
||||
} else {
|
||||
assert!(did.contains(&Did::StopAll), "{kind:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_lets_the_brain_get_ready_for_the_joined_request() {
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
|
||||
@@ -96,8 +96,10 @@ started within `call.continueMs` of the request's end (start = the final's arriv
|
||||
a stop (a stop stops, as always) and an empty final (noise the worker rejected: nothing is joined
|
||||
or asked again, the request goes on as it was).
|
||||
- When the speaker's next utterance starts (`SpeechStart`) within `call.continueMs` of the request's
|
||||
end, the request is held open for that utterance's final, also if its answer starts playing
|
||||
meanwhile; the joined request then stops that answer. An empty final ends the hold.
|
||||
end, or had already started when the request's final came (a pause shorter than the
|
||||
transcription), the request is held open for that utterance's final, also if its answer starts
|
||||
playing or even ends meanwhile; the joined request then stops that answer if it still plays. An
|
||||
empty final or a stop ends the hold, and nothing is asked again.
|
||||
- Other speakers' utterances never join it. An answer already heard, an answer that ended, a missed
|
||||
request, a stop, leaving and a lost worker end it; then what the speaker says is a new request.
|
||||
- The joined request gets no second earcon when the first one's already played.
|
||||
@@ -253,8 +255,9 @@ off, partials that read as a call sent to get ready (joined to a waiting wake wo
|
||||
answering someone else, the joined text while going on), going on (pieces joined into one answer in
|
||||
every policy without a missed record, a piece without a wake word joined, other speakers never, a
|
||||
new request once the answer is heard or the pause is longer than `call.continueMs`, a stop still
|
||||
stopping, no second earcon, an utterance started before the answer played still joined, an empty
|
||||
final joining nothing), interruption cancelling the right speech, a ranking deciding, end times from
|
||||
stopping, no second earcon, an utterance started before the answer played or before the first final
|
||||
came still joined, also after a short answer ended, in every policy; an empty final or a stop
|
||||
after a hold asking nothing again), interruption cancelling the right speech, a ranking deciding, end times from
|
||||
`wait_ms`, and chat: an answer posted as a reply once, per-channel switches, no answers to bots or to
|
||||
a stop, chat wake words and designated people, a failed answer posting nothing). `tests/worker_protocol.rs` checks an answer spoken under its id and both judgements against
|
||||
the real Python worker with a fake brain.
|
||||
|
||||
@@ -96,3 +96,8 @@
|
||||
- 빈 결과의 원인은 확신도 기준(-0.7). -0.85/-1.0으로 낮추면 헤드셋 7.5%, 잡음 10dB 15.5%, 5dB 30.2%, 빈 결과 원본 17→15/10개. 대신 잡음 108개(VAD 없이 바로 넣음)에서 글자가 나온 것이 16→27/32개. 실제 워커는 VAD가 먼저 거르므로(M1: 108개 중 5개만 모델까지 감) 기준 변경은 VAD를 포함한 잡음 시험으로 다시 재고 사용자가 정할 것. 이번에는 바꾸지 않음.
|
||||
- 작은 목소리+마이크 잡음(약 SNR 5dB)은 어떤 설정으로도 50% 안팎이라, 소리 앞단의 잡음 제거(모델) 같은 다른 방향이 필요함. 아직 미검증.
|
||||
- 근거: groups/javis_bot/evidence/2026-11-19/stt-plan/harsh/ (build.py 조건 정의, harsh.py, gate.py, 결과 JSON).
|
||||
- 2026-11-19 밤 (0.1.50): 악조건 두 갈래를 같은 자료(녹음 100개 × 7조건, 정답 16개)로 비교. 근거: evidence/2026-11-19/stt-plan/harsh2/.
|
||||
- 앞단 잡음 제거 3종(DeepFilterNet3, RNNoise, noisereduce)은 전부 나빠서 넣지 않음. 원본 7.0% → 13.8/9.6/11.0%, 싼 헤드셋 13.8% → 21.8/11.4/19.5%, 잡음 10dB 18.9% → 44.9/48.3/31.6%. DeepFilterNet은 작은 목소리에서 없는 말을 길게 지어냈고(108.6%), VAD를 지난 잡음 5개를 모두 글자로 만듦. 디스코드가 이미 보내는 쪽에서 잡음 제거(Krisp)를 하고, Whisper는 잡음 섞인 소리로 학습되어 제거 흔적이 더 해로움.
|
||||
- 확신도 기준: 실제 워커처럼 VAD와 분할기로 자른 잡음만 넣으면 모델까지 가는 것은 108개 중 5개(웃음). 기준을 -0.8로만 낮춰도 그중 1개가 "감사합니다."가 되고, VAD 없이 넣은 108개도 16→20개. 그래서 기준 자체는 -0.7 그대로.
|
||||
- 대신 살리기: 문맥 없는 결과가 -0.7에 조금 못 미쳐도(-0.8 이상) 문맥을 준 결과가 기준을 넘고 가까우면 그것을 씀. 실제 코드로 다시 잼(verify.py): 원본 7.0%, 헤드셋 13.8→7.5%, 빈 결과 대부분 1~2개 줄어듦, 잡음 0/5와 16/108 그대로, 최종 전사 평균 178~180ms(p95 246~254ms), 10명 한 묶음 1.1초.
|
||||
- 남은 문제(수치 그대로): 작은 목소리+마이크 잡음 51.4%(빈 결과 48/100), 잡음 5dB 33.9%(34/100), 잡음 10dB 18.9%(23/100). 이번에 본 방법(모델 7종, 문맥, 기준, 잡음 제거 3종)으로는 나아지지 않음. 다음 후보는 이런 조건의 실제 녹음으로 미세조정한 모델이나, 보내는 사람 쪽 디스코드 설정(입력 감도, 잡음 제거) 안내.
|
||||
|
||||
@@ -19,9 +19,13 @@ SR = 16000
|
||||
# over it: every hallucination stayed below, real speech above except two short words in loud noise.
|
||||
# Whisper's no_speech_prob is not used: large-v3-turbo reports 0.0 for silence and noise alike.
|
||||
MIN_MEAN_LOGPROB = -0.7
|
||||
MAX_COMPRESSION_RATIO = 2.4
|
||||
MAX_COMPRESSION_RATIO = 2.4 # repetition loops ("A B A B A B ...") compress far better than speech
|
||||
# faster-whisper keeps at most this many previous-text tokens in a prompt (max_length // 2 - 1).
|
||||
MAX_PREVIOUS_TOKENS = 223 # repetition loops ("A B A B A B ...") compress far better than speech
|
||||
MAX_PREVIOUS_TOKENS = 223
|
||||
# A plain decode at or above this, though below MIN_MEAN_LOGPROB, may be rescued by the context
|
||||
# (choose_with_context). Measured 2026-11-19 on harsh copies of the user's speech: down to -0.8 no
|
||||
# more noise came out as text (VAD-cut noise 0/5, raw noise 16/108 as without); -0.9 let 3 more through.
|
||||
UNSURE_LOGPROB = -0.8
|
||||
|
||||
|
||||
def _letters(text: str) -> str:
|
||||
@@ -75,16 +79,24 @@ def choose_with_hint(hint_text: str, hint_lp: float, plain_text: str, plain_lp:
|
||||
return plain_text
|
||||
|
||||
|
||||
def choose_with_context(heard: str, prompted: str, hotwords: tuple[str, ...]) -> str:
|
||||
def choose_with_context(heard: str, prompted: str, hotwords: tuple[str, ...],
|
||||
plain: tuple[str, float] | None = None) -> str:
|
||||
"""Pick between the decode without the conversation context (``heard``, already through the hint
|
||||
choice) and the one that had it as Whisper's previous-text prompt (``prompted``).
|
||||
choice) and the one that had it as Whisper's previous-text prompt (``prompted``, through the same
|
||||
choice, so it passed the gate itself).
|
||||
|
||||
The context fixes words of the topic the model misheard, but on noise or unsure audio it can write
|
||||
words of the prompt, so the decode without it decides, as for the hint: nothing heard stays
|
||||
nothing, and the prompted text is used only when it stays close to what was heard and holds every
|
||||
hotword exactly as often.
|
||||
words of the prompt, so the decode without it decides, as for the hint: the prompted text is used
|
||||
only when it stays close to what was heard and holds every hotword exactly as often. When nothing
|
||||
was heard, ``plain`` (the plain decode's text and mean log-probability) may still be just short of
|
||||
the gate (at least UNSURE_LOGPROB): then the prompted text is used if it is that close to it.
|
||||
"""
|
||||
if not heard or not prompted:
|
||||
if not heard:
|
||||
if not prompted or plain is None or plain[1] < UNSURE_LOGPROB \
|
||||
or compression_ratio(plain[0]) > MAX_COMPRESSION_RATIO:
|
||||
return ""
|
||||
return prompted if choose_with_context(plain[0], prompted, hotwords) == prompted else ""
|
||||
if not prompted:
|
||||
return heard
|
||||
h, p = _letters(heard), _letters(prompted)
|
||||
if _edit_distance(p, h) > len(h) // 2:
|
||||
@@ -130,13 +142,14 @@ class WhisperTranscriber:
|
||||
"""Clips arrive loudness-equalised by the session. ``contexts``: per clip, the recent
|
||||
conversation to prompt with ("" for none); see ``choose_with_context``."""
|
||||
enc, langs = self._encode(clips, opts)
|
||||
heard = self._choose(enc, langs, opts, None)
|
||||
plain = self._decode(enc, langs, opts.beam, None)
|
||||
heard = self._choose(enc, langs, opts, None, plain)
|
||||
prev = self._previous(langs, contexts)
|
||||
if prev is None:
|
||||
return heard
|
||||
prompted = self._choose(enc, langs, opts, prev)
|
||||
return [choose_with_context(h, p, opts.hotwords) if c else h
|
||||
for h, p, c in zip(heard, prompted, contexts)]
|
||||
prompted = self._choose(enc, langs, opts, prev, self._decode(enc, langs, opts.beam, None, prev))
|
||||
return [choose_with_context(h, p, opts.hotwords, pl) if c else h
|
||||
for h, p, pl, c in zip(heard, prompted, plain, contexts)]
|
||||
|
||||
def _previous(self, langs: list[str], contexts: list[str] | None) -> list[list[int]] | None:
|
||||
"""The contexts as previous-text tokens of one length for the whole batch: the decoder needs
|
||||
@@ -166,8 +179,8 @@ class WhisperTranscriber:
|
||||
langs = [r[0][0][2:-2] for r in m.model.detect_language(enc)]
|
||||
return enc, langs
|
||||
|
||||
def _choose(self, enc, langs: list[str], opts: DecodeOptions, prev: list[list[int]] | None) -> list[str]:
|
||||
plain = self._decode(enc, langs, opts.beam, None, prev)
|
||||
def _choose(self, enc, langs: list[str], opts: DecodeOptions, prev: list[list[int]] | None,
|
||||
plain: list[tuple[str, float]]) -> list[str]:
|
||||
if not opts.hotwords:
|
||||
return [text if keep_transcript(text, lp) else "" for text, lp in plain]
|
||||
hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords), prev)
|
||||
|
||||
@@ -171,6 +171,12 @@ decoded with the recent conversation as its previous-text prompt:
|
||||
was kept, it stays within half that text's letters of it, and it holds every hotword exactly as
|
||||
often; otherwise the text without context is used. On noise or unsure audio the prompt makes
|
||||
Whisper write words of the conversation; this rule keeps that out, as for the hint.
|
||||
- One rescue: when the decode without context was dropped by the gate but its mean log-probability
|
||||
is at least -0.8 (just short of -0.7), the prompted text (which passed the gate itself) is used if
|
||||
it is that close to it and keeps every hotword count. The unsure text itself is never used.
|
||||
Measured on harsh copies of the user's speech: a cheap headset 13.8% to 7.5% error, empty results
|
||||
fewer by 1-2 per 100 in most conditions, and no more noise with text (VAD-cut noise 0 of 5, noise
|
||||
decoded straight 16 of 108, as before); -0.9 let 3 more noise clips through.
|
||||
- Finals stay batched. The decoder needs the start token at the same position in every prompt of a
|
||||
batch, so each clip's context is cut from its end to the shortest one in the batch (a channel's
|
||||
clips share most of it); a clip without a context borrows another's and keeps its plain result.
|
||||
|
||||
@@ -132,3 +132,32 @@ def test_the_context_never_changes_how_many_wake_words_were_heard():
|
||||
@pytest.mark.unit
|
||||
def test_an_empty_prompted_decode_keeps_what_was_heard():
|
||||
assert choose_with_context("네.", "", HINT) == "네."
|
||||
|
||||
|
||||
# A plain decode just short of the gate (live 2026-11-19 on a headset copy: "니코 어디갔어" at -0.72 for
|
||||
# "디코 어디 갔어") is rescued by a confident prompted decode close to it; nothing else changes.
|
||||
from voice_worker.stt_whisper import UNSURE_LOGPROB # noqa: E402
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_nearly_sure_decode_is_rescued_by_a_confident_close_prompted_one():
|
||||
assert choose_with_context("", "디코 어디 갔어?", HINT, ("니코 어디갔어", MIN_MEAN_LOGPROB - 0.02)) \
|
||||
== "디코 어디 갔어?"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_an_unsure_decode_well_below_the_gate_is_not_rescued():
|
||||
assert choose_with_context("", "디코 어디 갔어?", HINT, ("니코 어디갔어", UNSURE_LOGPROB - 0.05)) == ""
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_noise_with_a_prompt_far_from_it_is_not_rescued():
|
||||
# The prompted decode wrote words of the conversation over laughter.
|
||||
assert choose_with_context("", "화면에 디스코드", HINT, ("감사합니다.", MIN_MEAN_LOGPROB - 0.05)) == ""
|
||||
assert choose_with_context("", "", HINT, ("감사합니다.", MIN_MEAN_LOGPROB - 0.05)) == ""
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_rescue_never_adds_a_wake_word():
|
||||
assert choose_with_context("", "자비스 지금 영상 재생되고 있어?", HINT,
|
||||
("라비스 지금 영상 재생되고 있어", MIN_MEAN_LOGPROB - 0.05)) == ""
|
||||
|
||||
Reference in New Issue
Block a user