Compare commits
8 Commits
app-v0.1.4
...
app-v0.1.5
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
41f539dd48 | ||
|
|
b30aa4a0db | ||
|
|
835b547ca2 | ||
|
|
09bdf05762 | ||
|
|
251f5e116d | ||
|
|
c1b561d66d | ||
|
|
9a527aa470 | ||
|
|
628d89c4f1 |
2
app/Cargo.lock
generated
2
app/Cargo.lock
generated
@@ -1520,7 +1520,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.47"
|
||||
version = "0.1.50"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.47"
|
||||
version = "0.1.50"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"
|
||||
|
||||
4
app/shell/Cargo.lock
generated
4
app/shell/Cargo.lock
generated
@@ -2637,7 +2637,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.47"
|
||||
version = "0.1.50"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -2666,7 +2666,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.47"
|
||||
version = "0.1.50"
|
||||
dependencies = [
|
||||
"jarvis-app",
|
||||
"serde_json",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.47"
|
||||
version = "0.1.50"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"
|
||||
|
||||
@@ -179,6 +179,11 @@ struct Turn {
|
||||
ended_at: Instant,
|
||||
/// Still a "was it said to Jarvis?" question (its ask id), not yet a request of the policy.
|
||||
ask: Option<String>,
|
||||
/// The speaker started their next utterance within `call.continueMs` while the answer was not
|
||||
/// heard: the request stays open for that utterance's final, also if the answer starts meanwhile.
|
||||
held: bool,
|
||||
/// Its answer started playing (kept only while `held`).
|
||||
heard: bool,
|
||||
}
|
||||
|
||||
impl<V: Voice> Conversation<V> {
|
||||
@@ -274,6 +279,10 @@ impl<V: Voice> Conversation<V> {
|
||||
if let Some((_, deadline)) = self.wakes.get_mut(&h.speaker_id) {
|
||||
*deadline = None; // they go on: wait for this utterance's final
|
||||
}
|
||||
let window = self.calls.continue_window;
|
||||
if let Some(t) = self.turns.get_mut(&h.speaker_id) {
|
||||
t.held = !window.is_zero() && now <= t.ended_at + window;
|
||||
}
|
||||
}
|
||||
CoreEvent::Chat(m) => self.on_chat(m, now),
|
||||
CoreEvent::Partial(h, partial) => self.on_partial(h.speaker_id, partial),
|
||||
@@ -352,7 +361,7 @@ impl<V: Voice> Conversation<V> {
|
||||
let id = format!("q{}", req.id);
|
||||
if self.voice.ask_addressed(&id, speaker, &text, room) {
|
||||
self.turns.insert(speaker, Turn { id: req.id, text: req.text.clone(),
|
||||
ended_at: req.ended_at, ask: Some(id.clone()) });
|
||||
ended_at: req.ended_at, ask: Some(id.clone()), held: self.holds(speaker), heard: false });
|
||||
self.asks.insert(id, (req, now));
|
||||
} else {
|
||||
let a = self.policy.utterance_closed(speaker, now);
|
||||
@@ -384,13 +393,20 @@ impl<V: Voice> Conversation<V> {
|
||||
// The answer is heard: what its asker says next is a new request.
|
||||
CoreEvent::SpeakStart(id) => {
|
||||
if let Some(pid) = self.by_speech.get(id).copied() {
|
||||
self.turns.retain(|_, t| t.id != pid);
|
||||
self.turns.retain(|_, t| t.id != pid || t.held);
|
||||
for t in self.turns.values_mut().filter(|t| t.id == pid) {
|
||||
t.heard = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
CoreEvent::SpeakEnd(e) => {
|
||||
if let Some(pid) = self.by_speech.remove(&e.id) {
|
||||
self.answers.remove(&pid);
|
||||
self.turns.retain(|_, t| t.id != pid);
|
||||
// A held request waits for its speaker's next final even after its answer ended.
|
||||
self.turns.retain(|_, t| t.id != pid || t.held);
|
||||
for t in self.turns.values_mut().filter(|t| t.id == pid) {
|
||||
t.heard = true;
|
||||
}
|
||||
debug_log!("conversation", "answer req={pid} done played={} cancelled={} failed={}",
|
||||
e.played, e.cancelled, e.failed);
|
||||
let a = self.policy.answer_done(pid, now);
|
||||
@@ -417,9 +433,17 @@ impl<V: Voice> Conversation<V> {
|
||||
}
|
||||
}
|
||||
|
||||
/// The speaker's next utterance already started (a pause shorter than the first final's
|
||||
/// transcription): their new request is held open for it from the start.
|
||||
fn holds(&self, speaker: u64) -> bool {
|
||||
!self.calls.continue_window.is_zero() && self.started.contains_key(&speaker)
|
||||
}
|
||||
|
||||
/// A voice request goes to the policy; until its answer is heard, its speaker may go on.
|
||||
fn voice_request(&mut self, req: Request, now: Instant) {
|
||||
self.turns.insert(req.speaker, Turn { id: req.id, text: req.text.clone(), ended_at: req.ended_at, ask: None });
|
||||
let held = self.holds(req.speaker);
|
||||
self.turns.insert(req.speaker, Turn { id: req.id, text: req.text.clone(), ended_at: req.ended_at, ask: None,
|
||||
held, heard: false });
|
||||
let a = self.policy.request(req, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
@@ -428,14 +452,21 @@ impl<V: Voice> Conversation<V> {
|
||||
/// `call.continueMs` of that request's end, and is not a stop): the request is taken back, its
|
||||
/// running answer stopped. Returns its text and whether its earcon already played.
|
||||
fn going_on(&mut self, speaker: u64, f: &crate::discord::HeardFinal, now: Instant) -> Option<(String, bool)> {
|
||||
let t = self.turns.get(&speaker)?;
|
||||
let t = self.turns.get_mut(&speaker)?;
|
||||
let started = now.checked_sub(Duration::from_millis(f.wait_ms + f.audio_ms)).unwrap_or(now);
|
||||
if self.calls.continue_window.is_zero() || started > t.ended_at + self.calls.continue_window
|
||||
// An empty final (noise the worker rejected) adds nothing and asks nothing again.
|
||||
if crate::calls::normalise(&f.text).is_empty() || self.calls.continue_window.is_zero()
|
||||
|| started > t.ended_at + self.calls.continue_window
|
||||
|| matches!(self.calls.judge(speaker, &f.text), Call::Stop) {
|
||||
t.held = false;
|
||||
if t.heard {
|
||||
self.turns.remove(&speaker);
|
||||
}
|
||||
return None;
|
||||
}
|
||||
let t = self.turns.remove(&speaker).expect("checked");
|
||||
let earcon_played = self.chimed.remove(&t.id) || (self.calls.earcon && self.answers.contains_key(&t.id));
|
||||
let earcon_played = self.chimed.remove(&t.id)
|
||||
|| (self.calls.earcon && (self.answers.contains_key(&t.id) || t.heard));
|
||||
debug_log!("conversation", "going on req={} speaker={speaker}", t.id);
|
||||
match t.ask {
|
||||
Some(q) => {
|
||||
@@ -1156,6 +1187,101 @@ mod tests {
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스 날씨 알려 줘".into()), Did::StopAll, Did::Cancel("s1".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_that_started_before_the_answer_played_is_still_joined() {
|
||||
// The next utterance starts while Jarvis thinks; the answer starts playing before its final.
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1500));
|
||||
c.on_event(&fin_utt(A, 2, "디코 화면 공유 창에 가려져 있잖아"), ms(t, 3000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into()), Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_final_leaves_the_request_as_it_is() {
|
||||
// Noise the worker rejected comes as an empty final: nothing to join, nothing asked again.
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
c.on_event(&fin(A, "", 100), ms(t, 1500));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into())]);
|
||||
// An empty final after the answer played ends the hold: what follows is a new request.
|
||||
c.on_event(&start(A, 2), ms(t, 2000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 2500));
|
||||
c.on_event(&fin_utt(A, 2, " "), ms(t, 3200));
|
||||
c.on_event(&fin(A, "고마워", 100), ms(t, 4500));
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s1".into()), Did::Answer(A, "고마워".into())]);
|
||||
}
|
||||
|
||||
const ALL: [PolicyKind; 4] = [PolicyKind::Latest, PolicyKind::InOrder, PolicyKind::First, PolicyKind::Important];
|
||||
|
||||
#[test]
|
||||
fn going_on_is_joined_even_when_the_short_answer_already_ended() {
|
||||
// Final -> next SpeechStart -> the answer plays and ends -> next Final.
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1500));
|
||||
c.on_event(&end("s1", false), ms(t, 2500));
|
||||
c.on_event(&fin_utt(A, 2, "디코 화면 공유 창에 가려져 있잖아"), ms(t, 4000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into()),
|
||||
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())], "{kind:?}");
|
||||
assert_eq!(c.missed().count(), 0, "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_that_started_before_the_first_final_is_joined() {
|
||||
// A very short pause: the next utterance starts before the first one's final arrives.
|
||||
for kind in ALL {
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&start(A, 1), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&fin_utt(A, 1, "지금 화면 봐봐"), ms(t, 1100));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1600));
|
||||
c.on_event(&fin_utt(A, 2, "디코 화면 공유 창에 가려져 있잖아"), ms(t, 3000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into()), Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())], "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_request_whose_next_final_is_empty_or_a_stop_is_not_asked_again() {
|
||||
for kind in ALL {
|
||||
for last in ["", "자비스 멈춰"] {
|
||||
// (a) the answer already ended.
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1500));
|
||||
c.on_event(&end("s1", false), ms(t, 2500));
|
||||
c.on_event(&fin_utt(A, 2, last), ms(t, 4000));
|
||||
let did = f.take();
|
||||
assert_eq!(did.iter().filter(|d| matches!(d, Did::Answer(..))).count(), 1, "{kind:?} {last:?}");
|
||||
// What follows is a new request, not joined to the old one.
|
||||
c.on_event(&fin(A, "고마워", 100), ms(t, 5000));
|
||||
assert_eq!(f.take().into_iter().filter(|d| matches!(d, Did::Answer(..))).collect::<Vec<_>>(),
|
||||
vec![Did::Answer(A, "고마워".into())], "{kind:?} {last:?}");
|
||||
// (b) the next utterance started before the first final, the answer is playing.
|
||||
let (mut c, f, t) = alone(kind);
|
||||
c.on_event(&start(A, 1), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&fin_utt(A, 1, "지금 화면 봐봐"), ms(t, 1100));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1600));
|
||||
c.on_event(&fin_utt(A, 2, last), ms(t, 3000));
|
||||
let did = f.take();
|
||||
assert_eq!(did.iter().filter(|d| matches!(d, Did::Answer(..))).count(), 1, "{kind:?} {last:?}");
|
||||
if last.is_empty() {
|
||||
assert!(!did.contains(&Did::Cancel("s1".into())), "noise does not stop the answer ({kind:?})");
|
||||
} else {
|
||||
assert!(did.contains(&Did::StopAll), "{kind:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_lets_the_brain_get_ready_for_the_joined_request() {
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
|
||||
@@ -93,7 +93,13 @@ started within `call.continueMs` of the request's end (start = the final's arriv
|
||||
- The joined text (the earlier text, a space, the new final) is judged and answered as one request,
|
||||
as if it had been said in one go; it ends when the new final ended. It may be joined again.
|
||||
- The new final is joined whatever it is on its own (a piece needs no wake word of its own), except
|
||||
a stop: a stop stops, as always.
|
||||
a stop (a stop stops, as always) and an empty final (noise the worker rejected: nothing is joined
|
||||
or asked again, the request goes on as it was).
|
||||
- When the speaker's next utterance starts (`SpeechStart`) within `call.continueMs` of the request's
|
||||
end, or had already started when the request's final came (a pause shorter than the
|
||||
transcription), the request is held open for that utterance's final, also if its answer starts
|
||||
playing or even ends meanwhile; the joined request then stops that answer if it still plays. An
|
||||
empty final or a stop ends the hold, and nothing is asked again.
|
||||
- Other speakers' utterances never join it. An answer already heard, an answer that ended, a missed
|
||||
request, a stop, leaving and a lost worker end it; then what the speaker says is a new request.
|
||||
- The joined request gets no second earcon when the first one's already played.
|
||||
@@ -224,7 +230,8 @@ one request, stop all, earcon, prepare, ask "addressed", ask "rank"), so the glu
|
||||
cannot be queued the answer is done at once); `StopAnswer` → `cancel` of that answer's speech;
|
||||
`Rank { id }` → `rank` with id `k<id>` and the candidates' request ids; `Missed` → a debug log and
|
||||
the missed list (the last 100: speaker, reason, time and length, never the text).
|
||||
- `SpeakStart` of an answer's speech: the answer is heard, its request is no longer open (Going on).
|
||||
- `SpeakStart` of an answer's speech: the answer is heard, its request is no longer open (Going on),
|
||||
unless its speaker already started their next utterance (held).
|
||||
- `SpeakEnd` of an answer's speech is that answer's done. The end of a speech the loop already
|
||||
cancelled starts nothing.
|
||||
- `Stopped` (all speech stopped from elsewhere: the slash command, `stop()`, the session ending) and
|
||||
@@ -248,7 +255,9 @@ off, partials that read as a call sent to get ready (joined to a waiting wake wo
|
||||
answering someone else, the joined text while going on), going on (pieces joined into one answer in
|
||||
every policy without a missed record, a piece without a wake word joined, other speakers never, a
|
||||
new request once the answer is heard or the pause is longer than `call.continueMs`, a stop still
|
||||
stopping, no second earcon), interruption cancelling the right speech, a ranking deciding, end times from
|
||||
stopping, no second earcon, an utterance started before the answer played or before the first final
|
||||
came still joined, also after a short answer ended, in every policy; an empty final or a stop
|
||||
after a hold asking nothing again), interruption cancelling the right speech, a ranking deciding, end times from
|
||||
`wait_ms`, and chat: an answer posted as a reply once, per-channel switches, no answers to bots or to
|
||||
a stop, chat wake words and designated people, a failed answer posting nothing). `tests/worker_protocol.rs` checks an answer spoken under its id and both judgements against
|
||||
the real Python worker with a fake brain.
|
||||
|
||||
@@ -82,3 +82,22 @@
|
||||
## 진행 기록
|
||||
|
||||
- 2026-11-19: 서버 GPU 사용 불가. 15:19 재부팅으로 커널 7.0.0-43이 올라왔는데 이 커널용 nvidia 모듈 패키지(`linux-modules-nvidia-595-open-7.0.0-43-generic`)가 설치되지 않음. 서버 측정은 CPU로만 가능.
|
||||
- 2026-11-19 저녁 (게임PC, 사용자가 "지금 컴퓨터 사용해도 됨"): 사용자가 내보낸 녹음과 대화 기록을 받음. 사용자 녹음 100개(11-16 11개, 11-18 53개, 11-19 36개), 정답 16개(11-16 시험 문장 11개 + 사용자가 준 5개).
|
||||
- 불만 2(끊어 말하기)의 원인은 STT가 아니었음: 한 생각 안의 쉼이 0.1~4.7초라 조각마다 따로 요청이 되었고, 앱이 새 조각으로 앞 답을 멈춰도 워커는 그 답을 끝까지 만들어 다음 답이 13~35초 기다림. 0.1.47에서 고침(conversation.spec.md "Going on", voice_worker.spec.md: 취소된 답은 시작하지 않거나 다음 모델·도구 호출 전에 멈춤).
|
||||
- H2: 녹음을 다시 봐도 소리 손상은 없었음(채널 64kbps, 4~7kHz 성분 정상). 빈 결과 17개는 모델이 확신하지 못해 버린 것(평균 확신도 -0.7 미만, "감사합니다" 같은 지어낸 말 포함).
|
||||
- H3 모델 비교(게임PC GPU, 워커와 같은 경로, int8_float16, 정답 16개 기준 문자 오류율): large-v3-turbo 12.0%, large-v3 12.7%, BuzzASR/korean 13.3%, batisay-ko-turbo 11.2%, ghost613 turbo-korean 33.1%. 모두 같은 곳을 틀림("기사야", "화면군이", "기구", "화면" 빠짐). 상대 20% 넘게 나은 모델이 없어서 모델은 바꾸지 않음. Qwen3-ASR-1.7B(Apache-2.0)는 서버 CPU에서 정답 16개만: 9.4%(한국어판 TeamUNIVA beta 11.7%), 같은 곳을 틀림("기사야", "공리", "가면", "기고"). 앞 대화 문맥을 준 turbo(7.5%)보다 못하고 bf16로 약 5GB가 필요해 Strata와 함께 남는 VRAM(약 2.5GB)에 들어가지 않아 바꾸지 않음.
|
||||
- H4 앞 대화 문맥: 12.0% → 7.5%(상대 38%), 빈 결과 그대로, 잡음 108개의 지어낸 말 그대로(16개), 최종 전사 평균 182ms(p95 253ms). 문맥만 주면 잡음에서 앞 대화 낱말을 씀(turbo 22개, large-v3 63개)이라 호출어 힌트와 같은 규칙으로 막음. 0.1.48에 넣음(voice_worker.spec.md "Conversation context").
|
||||
- 남은 것: 첫 낱말(호출어)과 짧은 말("기사야"→비서야, "기구"→디코)은 앞 대화가 없거나 너무 짧아 그대로. 호출 판정의 비슷한 말 허용이 호출어 쪽을 덮음.
|
||||
- 2026-11-19 밤 (검토 뒤): 0.1.49에서 고침
|
||||
- 이어 말하기: 다음 말이 시작된 뒤 답이 나오기 시작해도 그 말의 최종 결과까지 앞 요청을 붙잡아 둠. 빈 결과(잡음)는 합치지 않고 앞 요청을 다시 묻지 않음.
|
||||
- 취소: 답 시작, 도구 고르기, 계획, 계획한 도구 실행 직전에도 확인.
|
||||
- 문맥을 쓸 때도 여러 사람의 최종 전사를 한 번에 묶음(문맥을 묶음 안에서 가장 짧은 길이로 끝에서 맞춤). 10명 동시: 묶음 1.3초(문맥 없이 1.6~2.0초, Strata가 돌던 게임PC), VRAM 최대 15.7GB/16.3GB(시작 14.6GB).
|
||||
- B. 악조건(사용자 녹음 100개, 정답 16개, 문맥 없음 → 있음 문자 오류율): 원본 11.9→7.0%, 작은 목소리+마이크 잡음 52.8→51.4%, 잡음 SNR 10dB 18.8→18.9%, 5dB 34.2→33.9%, 싼 헤드셋 14.9→13.8%, Opus 12kbps 12.8→9.8%, 방 울림 13.0→8.8%. 문맥은 어느 조건에서도 나쁘게 하지 않음. 하지만 악조건 자체가 나쁨: 빈 결과가 17개에서 20~49개로 늘어남.
|
||||
- 빈 결과의 원인은 확신도 기준(-0.7). -0.85/-1.0으로 낮추면 헤드셋 7.5%, 잡음 10dB 15.5%, 5dB 30.2%, 빈 결과 원본 17→15/10개. 대신 잡음 108개(VAD 없이 바로 넣음)에서 글자가 나온 것이 16→27/32개. 실제 워커는 VAD가 먼저 거르므로(M1: 108개 중 5개만 모델까지 감) 기준 변경은 VAD를 포함한 잡음 시험으로 다시 재고 사용자가 정할 것. 이번에는 바꾸지 않음.
|
||||
- 작은 목소리+마이크 잡음(약 SNR 5dB)은 어떤 설정으로도 50% 안팎이라, 소리 앞단의 잡음 제거(모델) 같은 다른 방향이 필요함. 아직 미검증.
|
||||
- 근거: groups/javis_bot/evidence/2026-11-19/stt-plan/harsh/ (build.py 조건 정의, harsh.py, gate.py, 결과 JSON).
|
||||
- 2026-11-19 밤 (0.1.50): 악조건 두 갈래를 같은 자료(녹음 100개 × 7조건, 정답 16개)로 비교. 근거: evidence/2026-11-19/stt-plan/harsh2/.
|
||||
- 앞단 잡음 제거 3종(DeepFilterNet3, RNNoise, noisereduce)은 전부 나빠서 넣지 않음. 원본 7.0% → 13.8/9.6/11.0%, 싼 헤드셋 13.8% → 21.8/11.4/19.5%, 잡음 10dB 18.9% → 44.9/48.3/31.6%. DeepFilterNet은 작은 목소리에서 없는 말을 길게 지어냈고(108.6%), VAD를 지난 잡음 5개를 모두 글자로 만듦. 디스코드가 이미 보내는 쪽에서 잡음 제거(Krisp)를 하고, Whisper는 잡음 섞인 소리로 학습되어 제거 흔적이 더 해로움.
|
||||
- 확신도 기준: 실제 워커처럼 VAD와 분할기로 자른 잡음만 넣으면 모델까지 가는 것은 108개 중 5개(웃음). 기준을 -0.8로만 낮춰도 그중 1개가 "감사합니다."가 되고, VAD 없이 넣은 108개도 16→20개. 그래서 기준 자체는 -0.7 그대로.
|
||||
- 대신 살리기: 문맥 없는 결과가 -0.7에 조금 못 미쳐도(-0.8 이상) 문맥을 준 결과가 기준을 넘고 가까우면 그것을 씀. 실제 코드로 다시 잼(verify.py): 원본 7.0%, 헤드셋 13.8→7.5%, 빈 결과 대부분 1~2개 줄어듦, 잡음 0/5와 16/108 그대로, 최종 전사 평균 178~180ms(p95 246~254ms), 10명 한 묶음 1.1초.
|
||||
- 남은 문제(수치 그대로): 작은 목소리+마이크 잡음 51.4%(빈 결과 48/100), 잡음 5dB 33.9%(34/100), 잡음 10dB 18.9%(23/100). 이번에 본 방법(모델 7종, 문맥, 기준, 잡음 제거 3종)으로는 나아지지 않음. 다음 후보는 이런 조건의 실제 녹음으로 미세조정한 모델이나, 보내는 사람 쪽 디스코드 설정(입력 감도, 잡음 제거) 안내.
|
||||
|
||||
@@ -68,6 +68,14 @@ TRACE_RESULT_CHARS = 2000
|
||||
TRACE_THINKING_CHARS = 8000
|
||||
|
||||
|
||||
def _cancelled(before: str) -> bool:
|
||||
"""The answer is no longer wanted (turn_state.cancel): stop before the next model or tool call."""
|
||||
if _turn_state.cancelled():
|
||||
debug_log(f"reply cancelled before {before}", "planning")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _clip(text: str, limit: int) -> str:
|
||||
return text if len(text) <= limit else text[:limit] + "…"
|
||||
|
||||
@@ -1253,6 +1261,8 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
# Step 0.5: Deterministic on-screen site search. If the user named a site and
|
||||
# asked to search/open it while broadcasting, do it directly — the small chat
|
||||
# model otherwise just narrates without calling the browser tool.
|
||||
if _cancelled("it started"):
|
||||
return None
|
||||
_site_search_reply = _maybe_deterministic_site_search(text, db, cfg, language)
|
||||
if _site_search_reply is not None:
|
||||
return _site_search_reply
|
||||
@@ -1390,6 +1400,8 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
routed_tools = list(_cached_routed)
|
||||
debug_log("tool router served from hot-window cache", "planning")
|
||||
else:
|
||||
if _cancelled("the tool router"):
|
||||
return None
|
||||
routed_tools = _route_tools(cfg, redacted, _builtin_tools, mcp_tools, recent_messages)
|
||||
# Don't cache the router's "fall open to all tools" fallback. That
|
||||
# path fires when the LLM router times out, returns empty, or emits
|
||||
@@ -1532,6 +1544,8 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
if _one_step_tool:
|
||||
debug_log(f"intent judge: one-step tool {_one_step_tool}, no plan", "planning")
|
||||
elif not intent_judge or _turn_needs_tools:
|
||||
if _cancelled("the planner"):
|
||||
return None
|
||||
action_plan = plan_query(
|
||||
cfg=cfg,
|
||||
query=redacted,
|
||||
@@ -2376,8 +2390,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
_one_step_nudged = False
|
||||
|
||||
while turn < max_turns:
|
||||
if _turn_state.cancelled():
|
||||
debug_log("reply cancelled: no longer wanted", "planning")
|
||||
if _cancelled("a model call"):
|
||||
return None
|
||||
turn += 1
|
||||
debug_log(f"🔁 messages loop turn {turn}", "planning")
|
||||
@@ -2496,6 +2509,8 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
}
|
||||
],
|
||||
})
|
||||
if _cancelled("a planned tool call"):
|
||||
return None
|
||||
tools_started += 1
|
||||
_plan_result = run_tool_with_retries(
|
||||
db=db,
|
||||
@@ -2844,8 +2859,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
continue
|
||||
|
||||
# Execute tool (not for an answer that is no longer wanted: no actions on its behalf)
|
||||
if _turn_state.cancelled():
|
||||
debug_log("reply cancelled before a tool call", "planning")
|
||||
if _cancelled("a tool call"):
|
||||
return None
|
||||
tools_started += 1
|
||||
result = run_tool_with_retries(
|
||||
|
||||
@@ -32,7 +32,8 @@ class SttResult:
|
||||
|
||||
|
||||
class Transcriber(Protocol):
|
||||
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions) -> list[str]: ...
|
||||
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions,
|
||||
contexts: list[str] | None = None) -> list[str]: ...
|
||||
|
||||
|
||||
Callback = Callable[[SttResult], None]
|
||||
@@ -44,6 +45,7 @@ class _Job:
|
||||
audio: np.ndarray
|
||||
opts: DecodeOptions
|
||||
callback: Callback
|
||||
context: str = "" # finals only: the recent conversation, as the model's prompt
|
||||
|
||||
|
||||
class SttScheduler:
|
||||
@@ -74,8 +76,9 @@ class SttScheduler:
|
||||
pass
|
||||
self._executor.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
def submit_final(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback) -> None:
|
||||
self._finals.append(_Job(key, audio, opts, callback))
|
||||
def submit_final(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback,
|
||||
context: str = "") -> None:
|
||||
self._finals.append(_Job(key, audio, opts, callback, context))
|
||||
self._wake.set()
|
||||
|
||||
def submit_partial(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback) -> None:
|
||||
@@ -121,7 +124,7 @@ class SttScheduler:
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
texts = await loop.run_in_executor(self._executor, self.transcriber.transcribe, clips,
|
||||
batch[0].opts)
|
||||
batch[0].opts, [j.context for j in batch])
|
||||
error = None
|
||||
except Exception as e: # the model failing must not stop every speaker's STT
|
||||
texts, error = [""] * len(batch), f"{type(e).__name__}: {e}"
|
||||
|
||||
@@ -10,6 +10,7 @@ import itertools
|
||||
import json
|
||||
import secrets
|
||||
import time
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable
|
||||
|
||||
@@ -28,6 +29,7 @@ from .tts import TtsEngine, TtsRequest, TtsService
|
||||
CLOSE_BAD_HELLO = 4400
|
||||
CLOSE_BAD_TOKEN = 4401
|
||||
MAX_SPEAKERS = 32
|
||||
CONTEXT_SECONDS = 180 # how far back the STT's conversation context reaches
|
||||
|
||||
|
||||
def is_loopback(host: str) -> bool:
|
||||
@@ -274,6 +276,9 @@ class Session:
|
||||
self.chat_waiting: dict[str, asyncio.Future] = {}
|
||||
self._chat_ids = itertools.count(1)
|
||||
self.names: dict[str, str] = {}
|
||||
# What was said and answered lately in Jarvis's voice channel: (time, text), the STT's
|
||||
# conversation context (voice_worker.spec.md, "Conversation context"). Never logged.
|
||||
self.lines: deque[tuple[float, str]] = deque(maxlen=50)
|
||||
self.speakers: dict[str, _Speaker] = {}
|
||||
self.tts_requests: dict[str, TtsRequest] = {}
|
||||
# With recordings on: each spoken answer's text and mono samples, kept until it is made and the
|
||||
@@ -364,6 +369,15 @@ class Session:
|
||||
samples = st.converter.push(pcm)
|
||||
self._handle_events(speaker, st, st.segmenter.push(samples, time.monotonic()))
|
||||
|
||||
def _context(self) -> str:
|
||||
"""The recent conversation for a final's prompt: the channel's lines of the last
|
||||
CONTEXT_SECONDS, at most `context_chars` characters from the end."""
|
||||
n = self.settings.context_chars
|
||||
if not n or not self.voice_channel:
|
||||
return ""
|
||||
since = time.monotonic() - CONTEXT_SECONDS
|
||||
return " ".join(t for at, t in self.lines if at >= since)[-n:].strip()
|
||||
|
||||
def _opts(self, beam: int) -> DecodeOptions:
|
||||
return DecodeOptions(beam=beam, language=self.settings.language, hotwords=self.settings.hotwords)
|
||||
|
||||
@@ -388,7 +402,8 @@ class Session:
|
||||
else:
|
||||
sched.submit_final((self.sid, speaker), loudness_equalise(ev.audio, ev.voiced),
|
||||
self._opts(self.settings.final_beam),
|
||||
lambda res, ev=ev, said=said: self._send_final(speaker, ev, res, said))
|
||||
lambda res, ev=ev, said=said: self._send_final(speaker, ev, res, said),
|
||||
context=self._context())
|
||||
|
||||
def _partial_cb(self, speaker: str, st: _Speaker, utt: int):
|
||||
def cb(res: SttResult) -> None:
|
||||
@@ -402,6 +417,8 @@ class Session:
|
||||
store = self.worker.history
|
||||
if store is not None and channel and res.text.strip():
|
||||
store.add_voice(channel, speaker, name, res.text, ts)
|
||||
if channel and channel == self.voice_channel and res.text.strip():
|
||||
self.lines.append((time.monotonic(), res.text.strip()))
|
||||
if self.worker.recorder is not None:
|
||||
self.worker.recorder.said(ts, speaker, name, channel, res.text, ev.audio, SR)
|
||||
self.send({"type": "final", "speaker": speaker, "utt": ev.utt, "text": res.text,
|
||||
@@ -499,6 +516,8 @@ class Session:
|
||||
rec["text"] += text
|
||||
while len(self.tts_recorded) > 50: # never reported as played: not kept forever
|
||||
self.tts_recorded.pop(next(iter(self.tts_recorded)))
|
||||
if self.voice_channel and text.strip():
|
||||
self.lines.append((time.monotonic(), text.strip()))
|
||||
tts.feed(req, text, more)
|
||||
|
||||
def _on_tts_cancel(self, msg: dict) -> None:
|
||||
@@ -523,6 +542,7 @@ class Session:
|
||||
if self.voice_channel:
|
||||
debug_log(f"session {self.sid} left voice channel {self.voice_channel}", "voice_worker")
|
||||
self.voice_channel = None
|
||||
self.lines.clear()
|
||||
|
||||
def _on_voice_join(self, msg: dict) -> None:
|
||||
got = self._strings(msg, "channel")
|
||||
|
||||
@@ -13,6 +13,7 @@ class SttSettings:
|
||||
final_beam: int = 2
|
||||
language: str | None = "ko"
|
||||
hotwords: tuple[str, ...] = ()
|
||||
context_chars: int = 200
|
||||
|
||||
def update(self, changes: dict) -> None:
|
||||
"""Apply ``changes``; raises ValueError and changes nothing if any value is bad."""
|
||||
@@ -52,6 +53,8 @@ def _check(key: str, value):
|
||||
if value is not None and (not isinstance(value, str) or not 2 <= len(value) <= 8):
|
||||
raise ValueError("language must be a language code or null")
|
||||
return value
|
||||
if key == "context_chars":
|
||||
return _int_in(key, value, 0, 500)
|
||||
if key == "hotwords":
|
||||
if (not isinstance(value, (list, tuple)) or len(value) > 50
|
||||
or not all(isinstance(w, str) and 0 < len(w) <= 64 for w in value)):
|
||||
|
||||
@@ -20,6 +20,12 @@ SR = 16000
|
||||
# Whisper's no_speech_prob is not used: large-v3-turbo reports 0.0 for silence and noise alike.
|
||||
MIN_MEAN_LOGPROB = -0.7
|
||||
MAX_COMPRESSION_RATIO = 2.4 # repetition loops ("A B A B A B ...") compress far better than speech
|
||||
# faster-whisper keeps at most this many previous-text tokens in a prompt (max_length // 2 - 1).
|
||||
MAX_PREVIOUS_TOKENS = 223
|
||||
# A plain decode at or above this, though below MIN_MEAN_LOGPROB, may be rescued by the context
|
||||
# (choose_with_context). Measured 2026-11-19 on harsh copies of the user's speech: down to -0.8 no
|
||||
# more noise came out as text (VAD-cut noise 0/5, raw noise 16/108 as without); -0.9 let 3 more through.
|
||||
UNSURE_LOGPROB = -0.8
|
||||
|
||||
|
||||
def _letters(text: str) -> str:
|
||||
@@ -73,6 +79,33 @@ def choose_with_hint(hint_text: str, hint_lp: float, plain_text: str, plain_lp:
|
||||
return plain_text
|
||||
|
||||
|
||||
def choose_with_context(heard: str, prompted: str, hotwords: tuple[str, ...],
|
||||
plain: tuple[str, float] | None = None) -> str:
|
||||
"""Pick between the decode without the conversation context (``heard``, already through the hint
|
||||
choice) and the one that had it as Whisper's previous-text prompt (``prompted``, through the same
|
||||
choice, so it passed the gate itself).
|
||||
|
||||
The context fixes words of the topic the model misheard, but on noise or unsure audio it can write
|
||||
words of the prompt, so the decode without it decides, as for the hint: the prompted text is used
|
||||
only when it stays close to what was heard and holds every hotword exactly as often. When nothing
|
||||
was heard, ``plain`` (the plain decode's text and mean log-probability) may still be just short of
|
||||
the gate (at least UNSURE_LOGPROB): then the prompted text is used if it is that close to it.
|
||||
"""
|
||||
if not heard:
|
||||
if not prompted or plain is None or plain[1] < UNSURE_LOGPROB \
|
||||
or compression_ratio(plain[0]) > MAX_COMPRESSION_RATIO:
|
||||
return ""
|
||||
return prompted if choose_with_context(plain[0], prompted, hotwords) == prompted else ""
|
||||
if not prompted:
|
||||
return heard
|
||||
h, p = _letters(heard), _letters(prompted)
|
||||
if _edit_distance(p, h) > len(h) // 2:
|
||||
return heard
|
||||
if any(_mentions(prompted, w) != _mentions(heard, w) for w in hotwords):
|
||||
return heard
|
||||
return prompted
|
||||
|
||||
|
||||
def add_cuda_dll_dirs() -> None:
|
||||
"""On Windows ctranslate2 loads cuBLAS/cuDNN from the PATH; the pip wheels put them in nvidia/*/bin."""
|
||||
if not hasattr(os, "add_dll_directory"):
|
||||
@@ -104,8 +137,34 @@ class WhisperTranscriber:
|
||||
def warm_up(self) -> None:
|
||||
self.transcribe([np.zeros(SR, dtype=np.float32)], DecodeOptions(beam=1, language="en", hotwords=()))
|
||||
|
||||
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions) -> list[str]:
|
||||
"""Clips arrive loudness-equalised by the session."""
|
||||
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions,
|
||||
contexts: list[str] | None = None) -> list[str]:
|
||||
"""Clips arrive loudness-equalised by the session. ``contexts``: per clip, the recent
|
||||
conversation to prompt with ("" for none); see ``choose_with_context``."""
|
||||
enc, langs = self._encode(clips, opts)
|
||||
plain = self._decode(enc, langs, opts.beam, None)
|
||||
heard = self._choose(enc, langs, opts, None, plain)
|
||||
prev = self._previous(langs, contexts)
|
||||
if prev is None:
|
||||
return heard
|
||||
prompted = self._choose(enc, langs, opts, prev, self._decode(enc, langs, opts.beam, None, prev))
|
||||
return [choose_with_context(h, p, opts.hotwords, pl) if c else h
|
||||
for h, p, pl, c in zip(heard, prompted, plain, contexts)]
|
||||
|
||||
def _previous(self, langs: list[str], contexts: list[str] | None) -> list[list[int]] | None:
|
||||
"""The contexts as previous-text tokens of one length for the whole batch: the decoder needs
|
||||
the start token at the same position in every prompt. Each is cut from its end to the
|
||||
shortest one (a channel's clips share most of their context); a clip without one borrows
|
||||
another's, and its prompted result is not used."""
|
||||
if not contexts or not any(contexts):
|
||||
return None
|
||||
toks = [self._tokenizer(lang).encode(" " + ctx.strip())[-MAX_PREVIOUS_TOKENS:] if ctx else None
|
||||
for lang, ctx in zip(langs, contexts)]
|
||||
n = min(len(t) for t in toks if t is not None)
|
||||
filler = next(t for t in toks if t is not None)[-n:]
|
||||
return [t[-n:] if t is not None else filler for t in toks]
|
||||
|
||||
def _encode(self, clips: list[np.ndarray], opts: DecodeOptions):
|
||||
m = self.model
|
||||
n_frames = m.feature_extractor.nb_max_frames
|
||||
feats = []
|
||||
@@ -118,16 +177,20 @@ class WhisperTranscriber:
|
||||
langs = [opts.language] * len(clips)
|
||||
else:
|
||||
langs = [r[0][0][2:-2] for r in m.model.detect_language(enc)]
|
||||
plain = self._decode(enc, langs, opts.beam, None)
|
||||
return enc, langs
|
||||
|
||||
def _choose(self, enc, langs: list[str], opts: DecodeOptions, prev: list[list[int]] | None,
|
||||
plain: list[tuple[str, float]]) -> list[str]:
|
||||
if not opts.hotwords:
|
||||
return [text if keep_transcript(text, lp) else "" for text, lp in plain]
|
||||
hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords))
|
||||
hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords), prev)
|
||||
return [choose_with_hint(ht, hlp, pt, plp, opts.hotwords) for (ht, hlp), (pt, plp) in zip(hinted, plain)]
|
||||
|
||||
def _decode(self, enc, langs: list[str], beam: int, hotwords: str | None) -> list[tuple[str, float]]:
|
||||
def _decode(self, enc, langs: list[str], beam: int, hotwords: str | None,
|
||||
prev: list[list[int]] | None = None) -> list[tuple[str, float]]:
|
||||
m = self.model
|
||||
prompts = [m.get_prompt(self._tokenizer(lang), [], without_timestamps=True, hotwords=hotwords)
|
||||
for lang in langs]
|
||||
prompts = [m.get_prompt(self._tokenizer(lang), p, without_timestamps=True, hotwords=hotwords)
|
||||
for lang, p in zip(langs, prev or [[]] * len(langs))]
|
||||
results = m.model.generate(enc, prompts, beam_size=beam, max_length=224, suppress_blank=True,
|
||||
suppress_tokens=[-1], return_scores=True)
|
||||
# ``scores`` is the sequence log-probability divided by its length: the mean per token.
|
||||
|
||||
@@ -80,6 +80,7 @@ STT settings (all optional in `hello` and `config`):
|
||||
| `end_silence_ms` | 300 | Trailing VAD silence that ends an utterance. Range 100-2000. |
|
||||
| `final_beam` | 2 | Beam size for finals (partials always use 1). Range 1-5. |
|
||||
| `language` | `"ko"` | Whisper language code, or `null` to detect per utterance. |
|
||||
| `context_chars` | 200 | Finals are also decoded with the recent conversation as Whisper's previous-text prompt, at most this many characters (Conversation context). `0` turns it off. Range 0-500. |
|
||||
| `hotwords` | `[]` | Words (wake words, names) passed to Whisper as a hint. The hint alone made Whisper write the wake word on laughter, so with hotwords every clip is also decoded without them and that plain decode decides (see Hallucination gate). |
|
||||
|
||||
### Utterance rules
|
||||
@@ -152,6 +153,46 @@ text is empty:
|
||||
|
||||
`no_speech_prob` is not used: large-v3-turbo reports 0.0 for silence and noise alike.
|
||||
|
||||
### Conversation context
|
||||
|
||||
People talk about one thing for a while, and the words of that topic ("디코", "화면 공유", "계산기")
|
||||
were the ones misheard (live 2026-11-19): Whisper heard a casual, unclear "화면 공유" as "화면공이"
|
||||
and "계산기" as "계산지", and five models (large-v3-turbo, large-v3 and three Korean fine-tunes) made
|
||||
the same mistakes. Whisper was trained to continue from the text before a clip, so a final is also
|
||||
decoded with the recent conversation as its previous-text prompt:
|
||||
|
||||
- The context is what was said (the session's non-empty finals, everyone's) and what Jarvis said
|
||||
(the text of every `tts` request) in the voice channel Jarvis is in, over the last 180 s, joined
|
||||
by spaces, at most `context_chars` characters from the end. Joining another channel or leaving
|
||||
clears it. It lives in memory only, is never logged and goes nowhere but the model's prompt.
|
||||
- Partials have none (they stay one cheap decode).
|
||||
- The clip is decoded as before (with the hotword choice above) and again with the context (the
|
||||
same choice, each decode prompted). The prompted text is used only when the one without context
|
||||
was kept, it stays within half that text's letters of it, and it holds every hotword exactly as
|
||||
often; otherwise the text without context is used. On noise or unsure audio the prompt makes
|
||||
Whisper write words of the conversation; this rule keeps that out, as for the hint.
|
||||
- One rescue: when the decode without context was dropped by the gate but its mean log-probability
|
||||
is at least -0.8 (just short of -0.7), the prompted text (which passed the gate itself) is used if
|
||||
it is that close to it and keeps every hotword count. The unsure text itself is never used.
|
||||
Measured on harsh copies of the user's speech: a cheap headset 13.8% to 7.5% error, empty results
|
||||
fewer by 1-2 per 100 in most conditions, and no more noise with text (VAD-cut noise 0 of 5, noise
|
||||
decoded straight 16 of 108, as before); -0.9 let 3 more noise clips through.
|
||||
- Finals stay batched. The decoder needs the start token at the same position in every prompt of a
|
||||
batch, so each clip's context is cut from its end to the shortest one in the batch (a channel's
|
||||
clips share most of it); a clip without a context borrows another's and keeps its plain result.
|
||||
|
||||
Measured (2026-11-19, the game PC's RTX 5070 Ti with Strata loaded, large-v3-turbo int8_float16,
|
||||
beam 2, the 100 recordings of the user's speech from 2026-11-16 to 19, 16 of them with the user's own
|
||||
text): character error rate 12.0% without and 7.5% with the context (wrong sentences 10 to 8), no more
|
||||
empty results (17 of 100 both ways), final decode 134 ms to 182 ms on average (p95 170 to 253 ms).
|
||||
On the 108 noise clips decoded straight (no VAD), as many came out with text with the context as
|
||||
without it (16 each); without the rule above it was 22 with turbo, and with large-v3 63, which wrote
|
||||
whole sentences of the prompt. Under harsh copies of the same recordings (a weak voice over a mic
|
||||
noise floor, ESC-50 noise at SNR 10 and 5 dB, a 300-3400 Hz headset, Opus 12 kbps, reverb) the
|
||||
context never made the error rate worse (it fell most with Opus and reverb). Ten finals at once stay
|
||||
one batch: 1.3 s with contexts, 1.6-2.0 s without (Strata running). Evidence and the model comparison:
|
||||
`docs/v2/stt-accuracy-plan.md`.
|
||||
|
||||
Measured (2026-11-11, RTX 5050) with 54 real noise recordings from ESC-50 (typing, mouse, breathing,
|
||||
coughing, laughing, crying, vacuum, ...), each at its own level and at -25 dBFS: of the 108 noise-only
|
||||
clips streamed through the worker with the wake word hint on, 103 never reached the model (the VAD
|
||||
|
||||
@@ -64,3 +64,74 @@ def test_an_answer_cancelled_during_a_model_call_runs_no_tool_for_it(mock_config
|
||||
# The model asks for a tool, but the answer was taken back meanwhile: the tool is not run.
|
||||
reply, calls, invoked = _run(mock_config, db, dialogue_memory, cancel_after_calls=1)
|
||||
assert reply is None and calls == 1 and invoked == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_planned_tool_is_not_run_when_the_answer_is_cancelled_while_its_step_is_resolved(
|
||||
mock_config, db, dialogue_memory):
|
||||
# Small models run planned steps directly: the step resolver (a model call) returns a tool call,
|
||||
# but the answer was taken back meanwhile.
|
||||
from jarvis.reply import engine as engine_mod
|
||||
from jarvis.reply import turn_state
|
||||
|
||||
mock_config.ollama_chat_model = "gemma4:e2b"
|
||||
mock_config.evaluator_enabled = False
|
||||
cancelled = [False]
|
||||
invoked = []
|
||||
|
||||
def fake_resolve(*args, **kwargs):
|
||||
cancelled[0] = True
|
||||
return ("webSearch", {"query": "x"})
|
||||
|
||||
plan = ["webSearch query='a'", "webSearch query='b from step 1'", "Reply to the user."]
|
||||
turn_state.reset()
|
||||
turn_state.set_cancel(lambda: cancelled[0])
|
||||
try:
|
||||
with patch.object(engine_mod, "run_tool_with_retries", side_effect=lambda *a, **k: invoked.append(1)), \
|
||||
patch.object(engine_mod, "chat_with_messages", side_effect=AssertionError("no model call")), \
|
||||
patch.object(engine_mod, "select_tools", return_value=["webSearch", "stop"]), \
|
||||
patch.object(engine_mod, "extract_search_params_for_memory", return_value={"keywords": []}), \
|
||||
patch.object(engine_mod, "plan_query", return_value=plan), \
|
||||
patch.object(engine_mod, "_resolve_plan_step", side_effect=fake_resolve):
|
||||
reply = engine_mod.run_reply_engine(db=db, cfg=mock_config, tts=None, text="a then b",
|
||||
dialogue_memory=dialogue_memory)
|
||||
finally:
|
||||
turn_state.reset()
|
||||
assert reply is None and invoked == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@pytest.mark.parametrize("cancel_in", ["select_tools", "plan_query"])
|
||||
def test_a_cancel_during_routing_or_planning_stops_before_the_next_call(mock_config, db, dialogue_memory,
|
||||
cancel_in):
|
||||
from jarvis.reply import engine as engine_mod
|
||||
from jarvis.reply import turn_state
|
||||
|
||||
mock_config.ollama_chat_model = "gpt-oss:20b"
|
||||
mock_config.evaluator_enabled = False
|
||||
cancelled = [False]
|
||||
calls = []
|
||||
|
||||
def route(*a, **k):
|
||||
calls.append("route")
|
||||
cancelled[0] = cancel_in == "select_tools"
|
||||
return ["webSearch", "stop"]
|
||||
|
||||
def plan(*a, **k):
|
||||
calls.append("plan")
|
||||
cancelled[0] = True
|
||||
return ["webSearch query='a'", "Reply to the user."]
|
||||
|
||||
turn_state.reset()
|
||||
turn_state.set_cancel(lambda: cancelled[0])
|
||||
try:
|
||||
with patch.object(engine_mod, "chat_with_messages", side_effect=lambda *a, **k: calls.append("chat")), \
|
||||
patch.object(engine_mod, "run_tool_with_retries", side_effect=lambda *a, **k: calls.append("tool")), \
|
||||
patch.object(engine_mod, "select_tools", side_effect=route), \
|
||||
patch.object(engine_mod, "plan_query", side_effect=plan):
|
||||
reply = engine_mod.run_reply_engine(db=db, cfg=mock_config, tts=None, text="날씨 찾아 줘",
|
||||
dialogue_memory=dialogue_memory)
|
||||
finally:
|
||||
turn_state.reset()
|
||||
assert reply is None
|
||||
assert calls == (["route"] if cancel_in == "select_tools" else ["route", "plan"])
|
||||
|
||||
@@ -303,7 +303,7 @@ class SlowTranscriber(LevelTranscriber):
|
||||
super().__init__()
|
||||
self.delay = delay
|
||||
|
||||
def transcribe(self, clips, opts):
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
import time as _t
|
||||
_t.sleep(self.delay)
|
||||
return super().transcribe(clips, opts)
|
||||
|
||||
@@ -96,3 +96,68 @@ def test_wake_word_matching_ignores_case_spacing_and_punctuation():
|
||||
def test_spacing_and_punctuation_differences_do_not_change_the_choice():
|
||||
assert choose_with_hint("자비스 오늘 서울 날씨 어때", -0.1, "자비스, 오늘 서울 날씨 어때?", -0.1, HINT) \
|
||||
== "자비스 오늘 서울 날씨 어때"
|
||||
|
||||
|
||||
# The recent conversation as Whisper's previous-text prompt (voice_worker.spec.md, "Conversation
|
||||
# context"). Live 2026-11-19 the prompt fixed words of the topic ("화면공이" -> "화면 공유", "계산지" ->
|
||||
# "계산기"); on noise it wrote words of the prompt. The decode without it decides, as with the hint.
|
||||
from voice_worker.stt_whisper import choose_with_context # noqa: E402
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_context_may_fix_words_of_what_was_said():
|
||||
assert choose_with_context("내 화면공이 안 보여?", "내 화면 공유 안 보여?", HINT) == "내 화면 공유 안 보여?"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_nothing_heard_without_the_context_stays_nothing():
|
||||
# Noise: the prompted decode wrote words of the conversation.
|
||||
assert choose_with_context("", "화면에 디스코드", HINT) == ""
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_prompted_decode_far_from_what_was_heard_is_not_used():
|
||||
heard = "가상화면을 못 봐"
|
||||
copied = "네, 현재 화면에 디스코드 채팅 창과 여러 프로그램이 정상적으로 보입니다."
|
||||
assert choose_with_context(heard, copied, HINT) == heard
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_context_never_changes_how_many_wake_words_were_heard():
|
||||
assert choose_with_context("라비스 지금 영상 재생되고 있어?", "자비스 지금 영상 재생되고 있어?", HINT) \
|
||||
== "라비스 지금 영상 재생되고 있어?"
|
||||
assert choose_with_context("자비스 날씨 알려 줘", "날씨 알려 줘", HINT) == "자비스 날씨 알려 줘"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_an_empty_prompted_decode_keeps_what_was_heard():
|
||||
assert choose_with_context("네.", "", HINT) == "네."
|
||||
|
||||
|
||||
# A plain decode just short of the gate (live 2026-11-19 on a headset copy: "니코 어디갔어" at -0.72 for
|
||||
# "디코 어디 갔어") is rescued by a confident prompted decode close to it; nothing else changes.
|
||||
from voice_worker.stt_whisper import UNSURE_LOGPROB # noqa: E402
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_nearly_sure_decode_is_rescued_by_a_confident_close_prompted_one():
|
||||
assert choose_with_context("", "디코 어디 갔어?", HINT, ("니코 어디갔어", MIN_MEAN_LOGPROB - 0.02)) \
|
||||
== "디코 어디 갔어?"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_an_unsure_decode_well_below_the_gate_is_not_rescued():
|
||||
assert choose_with_context("", "디코 어디 갔어?", HINT, ("니코 어디갔어", UNSURE_LOGPROB - 0.05)) == ""
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_noise_with_a_prompt_far_from_it_is_not_rescued():
|
||||
# The prompted decode wrote words of the conversation over laughter.
|
||||
assert choose_with_context("", "화면에 디스코드", HINT, ("감사합니다.", MIN_MEAN_LOGPROB - 0.05)) == ""
|
||||
assert choose_with_context("", "", HINT, ("감사합니다.", MIN_MEAN_LOGPROB - 0.05)) == ""
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_rescue_never_adds_a_wake_word():
|
||||
assert choose_with_context("", "자비스 지금 영상 재생되고 있어?", HINT,
|
||||
("라비스 지금 영상 재생되고 있어", MIN_MEAN_LOGPROB - 0.05)) == ""
|
||||
|
||||
@@ -25,7 +25,7 @@ class FakeTranscriber:
|
||||
self.gate.set()
|
||||
self.fail_on_beam = fail_on_beam
|
||||
|
||||
def transcribe(self, clips, opts):
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
self.entered.set()
|
||||
self.gate.wait(5)
|
||||
self.calls.append((opts, [len(c) for c in clips]))
|
||||
@@ -218,12 +218,45 @@ class StepTranscriber:
|
||||
def step(self):
|
||||
self._steps.release()
|
||||
|
||||
def transcribe(self, clips, opts):
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
self.calls.append([len(c) for c in clips])
|
||||
self._steps.acquire(timeout=5)
|
||||
return ["" for _ in clips]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_each_final_reaches_the_model_with_its_own_context():
|
||||
class Ctx:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
self.calls.append(list(contexts or []))
|
||||
return [f"n{len(c)}" for c in clips]
|
||||
|
||||
eng = Ctx()
|
||||
|
||||
async def body():
|
||||
s = SttScheduler(eng)
|
||||
s.start()
|
||||
got = []
|
||||
done = asyncio.Event()
|
||||
|
||||
def cb(r):
|
||||
got.append(r.text)
|
||||
if len(got) == 3:
|
||||
done.set()
|
||||
s.submit_final("a", clip(100), FINAL, cb, context="화면 공유")
|
||||
s.submit_final("b", clip(200), FINAL, cb)
|
||||
s.submit_partial("c", clip(300), PARTIAL, cb)
|
||||
await asyncio.wait_for(done.wait(), 5)
|
||||
await s.stop()
|
||||
|
||||
asyncio.run(body())
|
||||
assert eng.calls[0] == ["화면 공유", ""]
|
||||
assert eng.calls[1] == [""], "partials go without context"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_partial_batches_are_kept_small_so_finals_can_cut_in():
|
||||
t = StepTranscriber()
|
||||
|
||||
@@ -32,7 +32,7 @@ class LevelTranscriber:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def transcribe(self, clips, opts):
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
self.calls.append((opts, len(clips)))
|
||||
return [f"level {np.argmax(np.abs(np.fft.rfft(c))) * 16000 / len(c) / 1000:.1f}" for c in clips]
|
||||
|
||||
@@ -336,7 +336,7 @@ def test_48k_stereo_input_reaches_the_model_at_16k():
|
||||
lengths = []
|
||||
orig = t.transcribe
|
||||
|
||||
def spy(clips, opts):
|
||||
def spy(clips, opts, contexts=None):
|
||||
lengths.extend(len(c) for c in clips)
|
||||
return orig(clips, opts)
|
||||
|
||||
@@ -496,3 +496,47 @@ def test_old_recordings_are_also_removed_while_the_worker_keeps_running(tmp_path
|
||||
r.said(later.timestamp(), "1", "A", "vc", "나중", np.zeros(1600))
|
||||
r.close()
|
||||
assert sorted(p.name for p in tmp_path.iterdir()) == [later.date().isoformat()]
|
||||
|
||||
|
||||
class ContextTranscriber(LevelTranscriber):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.contexts = []
|
||||
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
if opts.beam > 1: # finals
|
||||
self.contexts += list(contexts or [""] * len(clips))
|
||||
return super().transcribe(clips, opts)
|
||||
|
||||
|
||||
async def _said(ws, speaker, level):
|
||||
await stream(ws, speaker, level, 600)
|
||||
await ws.send(json.dumps({"type": "speaking_end", "speaker": speaker}))
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final" and m["speaker"] == speaker)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_each_final_is_transcribed_with_the_recent_conversation_of_the_channel():
|
||||
"""voice_worker.spec.md, "Conversation context": what was said and answered just before, in the
|
||||
channel Jarvis is in, is the model's prompt for the next final."""
|
||||
t = ContextTranscriber()
|
||||
|
||||
async def body(url):
|
||||
async with connect(url) as ws:
|
||||
await hello(ws, stt={"partials": False})
|
||||
await ws.send(json.dumps({"type": "voice_join", "channel": "vc1"}))
|
||||
await _said(ws, "100", 0.9)
|
||||
await ws.send(json.dumps({"type": "tts", "id": "r1", "text": "디스코드 화면 공유가 보입니다.",
|
||||
"more": False}))
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
|
||||
await _said(ws, "200", 0.6)
|
||||
await ws.send(json.dumps({"type": "voice_join", "channel": "vc2"})) # another channel
|
||||
await _said(ws, "100", 0.9)
|
||||
await ws.send(json.dumps({"type": "config", "context_chars": 0}))
|
||||
await _said(ws, "100", 0.6)
|
||||
|
||||
asyncio.run(with_worker(body, transcriber=t, tts=FakeTts()))
|
||||
assert t.contexts[0] == ""
|
||||
assert t.contexts[1] == "level 0.9 디스코드 화면 공유가 보입니다."
|
||||
assert t.contexts[2] == "", "a new channel starts without the old channel's conversation"
|
||||
assert t.contexts[3] == "", "context_chars 0 turns it off"
|
||||
|
||||
Reference in New Issue
Block a user