Compare commits

...

8 Commits

Author SHA1 Message Date
javis-bot
41f539dd48 chore(app): version 0.1.50
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 21:02:05 +09:00
javis-bot
b30aa4a0db feat(voice_worker): a plain decode just short of the gate is rescued by a confident close prompted one
Harsh copies of the user's speech (weak voice, noise at SNR 10/5 dB, headset band, Opus 12 kbps,
reverb) with the worker's own VAD for noise: lowering the gate itself let laughter through (VAD-cut
noise 0/5 -> 1/5 at -0.8), and three noise suppressors (DeepFilterNet3, RNNoise, noisereduce) made
every condition worse, so neither is used. Rescuing a plain decode at -0.8..-0.7 with the prompted
text when that passed the gate and stays close: headset 13.8% -> 7.5%, no more noise with text.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 21:02:05 +09:00
javis-bot
835b547ca2 fix(conversation): a held request survives its answer ending, and is held from the start when the next utterance already began
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 20:37:53 +09:00
javis-bot
09bdf05762 chore(app): version 0.1.49
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 20:30:41 +09:00
javis-bot
251f5e116d fix(conversation,voice_worker): going on survives the answer starting, ignores empty finals; cancel checks before every model call; context finals stay batched
- A speaker who starts their next utterance within call.continueMs holds their request open for
  that final, also if the answer starts playing meanwhile; an empty final (rejected noise) joins
  nothing and asks nothing again
- The engine checks the cancel before the tool router, the planner and a planned tool call too
- Finals with a context are batched again: each context is cut from its end to the shortest one in
  the batch (ctranslate2 needs the start token at one position); ten finals at once 1.3 s
- Measured harsh conditions (docs/v2/stt-accuracy-plan.md): the context never made them worse

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 20:30:29 +09:00
javis-bot
c1b561d66d docs(v2): STT plan, the Qwen3-ASR comparison
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 19:55:47 +09:00
javis-bot
9a527aa470 chore(app): version 0.1.48
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 19:41:46 +09:00
javis-bot
628d89c4f1 feat(voice_worker): finals are also decoded with the recent conversation as Whisper's prompt
The words of the topic were the ones misheard live (화면공이 for 화면 공유, 계산지 for 계산기), and
five models made the same mistakes. Each final is decoded as before and again with the last 180 s
of the channel's conversation (everyone's finals and Jarvis's speech, at most context_chars=200
characters) as the previous-text prompt; the prompted text is used only when the plain one was kept,
stays within half its letters and keeps every hotword count, as for the hint.

Measured on the game PC (RTX 5070 Ti, 100 recordings of the user, 16 with their own text): CER
12.0% -> 7.5%, empty results unchanged, noise clips with text unchanged (16/108), final decode
p95 170 -> 253 ms. Finals with a context are decoded one clip at a time (ctranslate2 needs the
start token at one position in a batch).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 19:38:10 +09:00
18 changed files with 547 additions and 36 deletions

2
app/Cargo.lock generated
View File

@@ -1520,7 +1520,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
[[package]]
name = "jarvis-app"
version = "0.1.47"
version = "0.1.50"
dependencies = [
"async-trait",
"base64 0.22.1",

View File

@@ -1,6 +1,6 @@
[package]
name = "jarvis-app"
version = "0.1.47"
version = "0.1.50"
edition = "2021"
rust-version = "1.85"
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"

4
app/shell/Cargo.lock generated
View File

@@ -2637,7 +2637,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
[[package]]
name = "jarvis-app"
version = "0.1.47"
version = "0.1.50"
dependencies = [
"async-trait",
"base64 0.22.1",
@@ -2666,7 +2666,7 @@ dependencies = [
[[package]]
name = "jarvis-shell"
version = "0.1.47"
version = "0.1.50"
dependencies = [
"jarvis-app",
"serde_json",

View File

@@ -1,6 +1,6 @@
[package]
name = "jarvis-shell"
version = "0.1.47"
version = "0.1.50"
edition = "2021"
rust-version = "1.85"
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"

View File

@@ -179,6 +179,11 @@ struct Turn {
ended_at: Instant,
/// Still a "was it said to Jarvis?" question (its ask id), not yet a request of the policy.
ask: Option<String>,
/// The speaker started their next utterance within `call.continueMs` while the answer was not
/// heard: the request stays open for that utterance's final, also if the answer starts meanwhile.
held: bool,
/// Its answer started playing (kept only while `held`).
heard: bool,
}
impl<V: Voice> Conversation<V> {
@@ -274,6 +279,10 @@ impl<V: Voice> Conversation<V> {
if let Some((_, deadline)) = self.wakes.get_mut(&h.speaker_id) {
*deadline = None; // they go on: wait for this utterance's final
}
let window = self.calls.continue_window;
if let Some(t) = self.turns.get_mut(&h.speaker_id) {
t.held = !window.is_zero() && now <= t.ended_at + window;
}
}
CoreEvent::Chat(m) => self.on_chat(m, now),
CoreEvent::Partial(h, partial) => self.on_partial(h.speaker_id, partial),
@@ -352,7 +361,7 @@ impl<V: Voice> Conversation<V> {
let id = format!("q{}", req.id);
if self.voice.ask_addressed(&id, speaker, &text, room) {
self.turns.insert(speaker, Turn { id: req.id, text: req.text.clone(),
ended_at: req.ended_at, ask: Some(id.clone()) });
ended_at: req.ended_at, ask: Some(id.clone()), held: self.holds(speaker), heard: false });
self.asks.insert(id, (req, now));
} else {
let a = self.policy.utterance_closed(speaker, now);
@@ -384,13 +393,20 @@ impl<V: Voice> Conversation<V> {
// The answer is heard: what its asker says next is a new request.
CoreEvent::SpeakStart(id) => {
if let Some(pid) = self.by_speech.get(id).copied() {
self.turns.retain(|_, t| t.id != pid);
self.turns.retain(|_, t| t.id != pid || t.held);
for t in self.turns.values_mut().filter(|t| t.id == pid) {
t.heard = true;
}
}
}
CoreEvent::SpeakEnd(e) => {
if let Some(pid) = self.by_speech.remove(&e.id) {
self.answers.remove(&pid);
self.turns.retain(|_, t| t.id != pid);
// A held request waits for its speaker's next final even after its answer ended.
self.turns.retain(|_, t| t.id != pid || t.held);
for t in self.turns.values_mut().filter(|t| t.id == pid) {
t.heard = true;
}
debug_log!("conversation", "answer req={pid} done played={} cancelled={} failed={}",
e.played, e.cancelled, e.failed);
let a = self.policy.answer_done(pid, now);
@@ -417,9 +433,17 @@ impl<V: Voice> Conversation<V> {
}
}
/// The speaker's next utterance already started (a pause shorter than the first final's
/// transcription): their new request is held open for it from the start.
fn holds(&self, speaker: u64) -> bool {
!self.calls.continue_window.is_zero() && self.started.contains_key(&speaker)
}
/// A voice request goes to the policy; until its answer is heard, its speaker may go on.
fn voice_request(&mut self, req: Request, now: Instant) {
self.turns.insert(req.speaker, Turn { id: req.id, text: req.text.clone(), ended_at: req.ended_at, ask: None });
let held = self.holds(req.speaker);
self.turns.insert(req.speaker, Turn { id: req.id, text: req.text.clone(), ended_at: req.ended_at, ask: None,
held, heard: false });
let a = self.policy.request(req, now);
self.apply(a, now);
}
@@ -428,14 +452,21 @@ impl<V: Voice> Conversation<V> {
/// `call.continueMs` of that request's end, and is not a stop): the request is taken back, its
/// running answer stopped. Returns its text and whether its earcon already played.
fn going_on(&mut self, speaker: u64, f: &crate::discord::HeardFinal, now: Instant) -> Option<(String, bool)> {
let t = self.turns.get(&speaker)?;
let t = self.turns.get_mut(&speaker)?;
let started = now.checked_sub(Duration::from_millis(f.wait_ms + f.audio_ms)).unwrap_or(now);
if self.calls.continue_window.is_zero() || started > t.ended_at + self.calls.continue_window
// An empty final (noise the worker rejected) adds nothing and asks nothing again.
if crate::calls::normalise(&f.text).is_empty() || self.calls.continue_window.is_zero()
|| started > t.ended_at + self.calls.continue_window
|| matches!(self.calls.judge(speaker, &f.text), Call::Stop) {
t.held = false;
if t.heard {
self.turns.remove(&speaker);
}
return None;
}
let t = self.turns.remove(&speaker).expect("checked");
let earcon_played = self.chimed.remove(&t.id) || (self.calls.earcon && self.answers.contains_key(&t.id));
let earcon_played = self.chimed.remove(&t.id)
|| (self.calls.earcon && (self.answers.contains_key(&t.id) || t.heard));
debug_log!("conversation", "going on req={} speaker={speaker}", t.id);
match t.ask {
Some(q) => {
@@ -1156,6 +1187,101 @@ mod tests {
assert_eq!(f.take(), vec![Did::Answer(A, "자비스 날씨 알려 줘".into()), Did::StopAll, Did::Cancel("s1".into())]);
}
#[test]
fn going_on_that_started_before_the_answer_played_is_still_joined() {
// The next utterance starts while Jarvis thinks; the answer starts playing before its final.
let (mut c, f, t) = alone(PolicyKind::Latest);
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
c.on_event(&start(A, 2), ms(t, 1000));
c.on_event(&speak_start("s1"), ms(t, 1500));
c.on_event(&fin_utt(A, 2, "디코 화면 공유 창에 가려져 있잖아"), ms(t, 3000));
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into()), Did::Cancel("s1".into()),
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())]);
}
#[test]
fn an_empty_final_leaves_the_request_as_it_is() {
// Noise the worker rejected comes as an empty final: nothing to join, nothing asked again.
let (mut c, f, t) = alone(PolicyKind::Latest);
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
c.on_event(&fin(A, "", 100), ms(t, 1500));
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into())]);
// An empty final after the answer played ends the hold: what follows is a new request.
c.on_event(&start(A, 2), ms(t, 2000));
c.on_event(&speak_start("s1"), ms(t, 2500));
c.on_event(&fin_utt(A, 2, " "), ms(t, 3200));
c.on_event(&fin(A, "고마워", 100), ms(t, 4500));
assert_eq!(f.take(), vec![Did::Cancel("s1".into()), Did::Answer(A, "고마워".into())]);
}
const ALL: [PolicyKind; 4] = [PolicyKind::Latest, PolicyKind::InOrder, PolicyKind::First, PolicyKind::Important];
#[test]
fn going_on_is_joined_even_when_the_short_answer_already_ended() {
// Final -> next SpeechStart -> the answer plays and ends -> next Final.
for kind in ALL {
let (mut c, f, t) = alone(kind);
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
c.on_event(&start(A, 2), ms(t, 1000));
c.on_event(&speak_start("s1"), ms(t, 1500));
c.on_event(&end("s1", false), ms(t, 2500));
c.on_event(&fin_utt(A, 2, "디코 화면 공유 창에 가려져 있잖아"), ms(t, 4000));
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into()),
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())], "{kind:?}");
assert_eq!(c.missed().count(), 0, "{kind:?}");
}
}
#[test]
fn going_on_that_started_before_the_first_final_is_joined() {
// A very short pause: the next utterance starts before the first one's final arrives.
for kind in ALL {
let (mut c, f, t) = alone(kind);
c.on_event(&start(A, 1), ms(t, 0));
c.on_event(&start(A, 2), ms(t, 1000));
c.on_event(&fin_utt(A, 1, "지금 화면 봐봐"), ms(t, 1100));
c.on_event(&speak_start("s1"), ms(t, 1600));
c.on_event(&fin_utt(A, 2, "디코 화면 공유 창에 가려져 있잖아"), ms(t, 3000));
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into()), Did::Cancel("s1".into()),
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())], "{kind:?}");
}
}
#[test]
fn a_held_request_whose_next_final_is_empty_or_a_stop_is_not_asked_again() {
for kind in ALL {
for last in ["", "자비스 멈춰"] {
// (a) the answer already ended.
let (mut c, f, t) = alone(kind);
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
c.on_event(&start(A, 2), ms(t, 1000));
c.on_event(&speak_start("s1"), ms(t, 1500));
c.on_event(&end("s1", false), ms(t, 2500));
c.on_event(&fin_utt(A, 2, last), ms(t, 4000));
let did = f.take();
assert_eq!(did.iter().filter(|d| matches!(d, Did::Answer(..))).count(), 1, "{kind:?} {last:?}");
// What follows is a new request, not joined to the old one.
c.on_event(&fin(A, "고마워", 100), ms(t, 5000));
assert_eq!(f.take().into_iter().filter(|d| matches!(d, Did::Answer(..))).collect::<Vec<_>>(),
vec![Did::Answer(A, "고마워".into())], "{kind:?} {last:?}");
// (b) the next utterance started before the first final, the answer is playing.
let (mut c, f, t) = alone(kind);
c.on_event(&start(A, 1), ms(t, 0));
c.on_event(&start(A, 2), ms(t, 1000));
c.on_event(&fin_utt(A, 1, "지금 화면 봐봐"), ms(t, 1100));
c.on_event(&speak_start("s1"), ms(t, 1600));
c.on_event(&fin_utt(A, 2, last), ms(t, 3000));
let did = f.take();
assert_eq!(did.iter().filter(|d| matches!(d, Did::Answer(..))).count(), 1, "{kind:?} {last:?}");
if last.is_empty() {
assert!(!did.contains(&Did::Cancel("s1".into())), "noise does not stop the answer ({kind:?})");
} else {
assert!(did.contains(&Did::StopAll), "{kind:?}");
}
}
}
}
#[test]
fn going_on_lets_the_brain_get_ready_for_the_joined_request() {
let (mut c, f, t) = alone(PolicyKind::Latest);

View File

@@ -93,7 +93,13 @@ started within `call.continueMs` of the request's end (start = the final's arriv
- The joined text (the earlier text, a space, the new final) is judged and answered as one request,
as if it had been said in one go; it ends when the new final ended. It may be joined again.
- The new final is joined whatever it is on its own (a piece needs no wake word of its own), except
a stop: a stop stops, as always.
a stop (a stop stops, as always) and an empty final (noise the worker rejected: nothing is joined
or asked again, the request goes on as it was).
- When the speaker's next utterance starts (`SpeechStart`) within `call.continueMs` of the request's
end, or had already started when the request's final came (a pause shorter than the
transcription), the request is held open for that utterance's final, also if its answer starts
playing or even ends meanwhile; the joined request then stops that answer if it still plays. An
empty final or a stop ends the hold, and nothing is asked again.
- Other speakers' utterances never join it. An answer already heard, an answer that ended, a missed
request, a stop, leaving and a lost worker end it; then what the speaker says is a new request.
- The joined request gets no second earcon when the first one's already played.
@@ -224,7 +230,8 @@ one request, stop all, earcon, prepare, ask "addressed", ask "rank"), so the glu
cannot be queued the answer is done at once); `StopAnswer` → `cancel` of that answer's speech;
`Rank { id }` → `rank` with id `k<id>` and the candidates' request ids; `Missed` → a debug log and
the missed list (the last 100: speaker, reason, time and length, never the text).
- `SpeakStart` of an answer's speech: the answer is heard, its request is no longer open (Going on).
- `SpeakStart` of an answer's speech: the answer is heard, its request is no longer open (Going on),
unless its speaker already started their next utterance (held).
- `SpeakEnd` of an answer's speech is that answer's done. The end of a speech the loop already
cancelled starts nothing.
- `Stopped` (all speech stopped from elsewhere: the slash command, `stop()`, the session ending) and
@@ -248,7 +255,9 @@ off, partials that read as a call sent to get ready (joined to a waiting wake wo
answering someone else, the joined text while going on), going on (pieces joined into one answer in
every policy without a missed record, a piece without a wake word joined, other speakers never, a
new request once the answer is heard or the pause is longer than `call.continueMs`, a stop still
stopping, no second earcon), interruption cancelling the right speech, a ranking deciding, end times from
stopping, no second earcon, an utterance started before the answer played or before the first final
came still joined, also after a short answer ended, in every policy; an empty final or a stop
after a hold asking nothing again), interruption cancelling the right speech, a ranking deciding, end times from
`wait_ms`, and chat: an answer posted as a reply once, per-channel switches, no answers to bots or to
a stop, chat wake words and designated people, a failed answer posting nothing). `tests/worker_protocol.rs` checks an answer spoken under its id and both judgements against
the real Python worker with a fake brain.

View File

@@ -82,3 +82,22 @@
## 진행 기록
- 2026-11-19: 서버 GPU 사용 불가. 15:19 재부팅으로 커널 7.0.0-43이 올라왔는데 이 커널용 nvidia 모듈 패키지(`linux-modules-nvidia-595-open-7.0.0-43-generic`)가 설치되지 않음. 서버 측정은 CPU로만 가능.
- 2026-11-19 저녁 (게임PC, 사용자가 "지금 컴퓨터 사용해도 됨"): 사용자가 내보낸 녹음과 대화 기록을 받음. 사용자 녹음 100개(11-16 11개, 11-18 53개, 11-19 36개), 정답 16개(11-16 시험 문장 11개 + 사용자가 준 5개).
- 불만 2(끊어 말하기)의 원인은 STT가 아니었음: 한 생각 안의 쉼이 0.1~4.7초라 조각마다 따로 요청이 되었고, 앱이 새 조각으로 앞 답을 멈춰도 워커는 그 답을 끝까지 만들어 다음 답이 13~35초 기다림. 0.1.47에서 고침(conversation.spec.md "Going on", voice_worker.spec.md: 취소된 답은 시작하지 않거나 다음 모델·도구 호출 전에 멈춤).
- H2: 녹음을 다시 봐도 소리 손상은 없었음(채널 64kbps, 4~7kHz 성분 정상). 빈 결과 17개는 모델이 확신하지 못해 버린 것(평균 확신도 -0.7 미만, "감사합니다" 같은 지어낸 말 포함).
- H3 모델 비교(게임PC GPU, 워커와 같은 경로, int8_float16, 정답 16개 기준 문자 오류율): large-v3-turbo 12.0%, large-v3 12.7%, BuzzASR/korean 13.3%, batisay-ko-turbo 11.2%, ghost613 turbo-korean 33.1%. 모두 같은 곳을 틀림("기사야", "화면군이", "기구", "화면" 빠짐). 상대 20% 넘게 나은 모델이 없어서 모델은 바꾸지 않음. Qwen3-ASR-1.7B(Apache-2.0)는 서버 CPU에서 정답 16개만: 9.4%(한국어판 TeamUNIVA beta 11.7%), 같은 곳을 틀림("기사야", "공리", "가면", "기고"). 앞 대화 문맥을 준 turbo(7.5%)보다 못하고 bf16로 약 5GB가 필요해 Strata와 함께 남는 VRAM(약 2.5GB)에 들어가지 않아 바꾸지 않음.
- H4 앞 대화 문맥: 12.0% → 7.5%(상대 38%), 빈 결과 그대로, 잡음 108개의 지어낸 말 그대로(16개), 최종 전사 평균 182ms(p95 253ms). 문맥만 주면 잡음에서 앞 대화 낱말을 씀(turbo 22개, large-v3 63개)이라 호출어 힌트와 같은 규칙으로 막음. 0.1.48에 넣음(voice_worker.spec.md "Conversation context").
- 남은 것: 첫 낱말(호출어)과 짧은 말("기사야"→비서야, "기구"→디코)은 앞 대화가 없거나 너무 짧아 그대로. 호출 판정의 비슷한 말 허용이 호출어 쪽을 덮음.
- 2026-11-19 밤 (검토 뒤): 0.1.49에서 고침
- 이어 말하기: 다음 말이 시작된 뒤 답이 나오기 시작해도 그 말의 최종 결과까지 앞 요청을 붙잡아 둠. 빈 결과(잡음)는 합치지 않고 앞 요청을 다시 묻지 않음.
- 취소: 답 시작, 도구 고르기, 계획, 계획한 도구 실행 직전에도 확인.
- 문맥을 쓸 때도 여러 사람의 최종 전사를 한 번에 묶음(문맥을 묶음 안에서 가장 짧은 길이로 끝에서 맞춤). 10명 동시: 묶음 1.3초(문맥 없이 1.6~2.0초, Strata가 돌던 게임PC), VRAM 최대 15.7GB/16.3GB(시작 14.6GB).
- B. 악조건(사용자 녹음 100개, 정답 16개, 문맥 없음 → 있음 문자 오류율): 원본 11.9→7.0%, 작은 목소리+마이크 잡음 52.8→51.4%, 잡음 SNR 10dB 18.8→18.9%, 5dB 34.2→33.9%, 싼 헤드셋 14.9→13.8%, Opus 12kbps 12.8→9.8%, 방 울림 13.0→8.8%. 문맥은 어느 조건에서도 나쁘게 하지 않음. 하지만 악조건 자체가 나쁨: 빈 결과가 17개에서 20~49개로 늘어남.
- 빈 결과의 원인은 확신도 기준(-0.7). -0.85/-1.0으로 낮추면 헤드셋 7.5%, 잡음 10dB 15.5%, 5dB 30.2%, 빈 결과 원본 17→15/10개. 대신 잡음 108개(VAD 없이 바로 넣음)에서 글자가 나온 것이 16→27/32개. 실제 워커는 VAD가 먼저 거르므로(M1: 108개 중 5개만 모델까지 감) 기준 변경은 VAD를 포함한 잡음 시험으로 다시 재고 사용자가 정할 것. 이번에는 바꾸지 않음.
- 작은 목소리+마이크 잡음(약 SNR 5dB)은 어떤 설정으로도 50% 안팎이라, 소리 앞단의 잡음 제거(모델) 같은 다른 방향이 필요함. 아직 미검증.
- 근거: groups/javis_bot/evidence/2026-11-19/stt-plan/harsh/ (build.py 조건 정의, harsh.py, gate.py, 결과 JSON).
- 2026-11-19 밤 (0.1.50): 악조건 두 갈래를 같은 자료(녹음 100개 × 7조건, 정답 16개)로 비교. 근거: evidence/2026-11-19/stt-plan/harsh2/.
- 앞단 잡음 제거 3종(DeepFilterNet3, RNNoise, noisereduce)은 전부 나빠서 넣지 않음. 원본 7.0% → 13.8/9.6/11.0%, 싼 헤드셋 13.8% → 21.8/11.4/19.5%, 잡음 10dB 18.9% → 44.9/48.3/31.6%. DeepFilterNet은 작은 목소리에서 없는 말을 길게 지어냈고(108.6%), VAD를 지난 잡음 5개를 모두 글자로 만듦. 디스코드가 이미 보내는 쪽에서 잡음 제거(Krisp)를 하고, Whisper는 잡음 섞인 소리로 학습되어 제거 흔적이 더 해로움.
- 확신도 기준: 실제 워커처럼 VAD와 분할기로 자른 잡음만 넣으면 모델까지 가는 것은 108개 중 5개(웃음). 기준을 -0.8로만 낮춰도 그중 1개가 "감사합니다."가 되고, VAD 없이 넣은 108개도 16→20개. 그래서 기준 자체는 -0.7 그대로.
- 대신 살리기: 문맥 없는 결과가 -0.7에 조금 못 미쳐도(-0.8 이상) 문맥을 준 결과가 기준을 넘고 가까우면 그것을 씀. 실제 코드로 다시 잼(verify.py): 원본 7.0%, 헤드셋 13.8→7.5%, 빈 결과 대부분 1~2개 줄어듦, 잡음 0/5와 16/108 그대로, 최종 전사 평균 178~180ms(p95 246~254ms), 10명 한 묶음 1.1초.
- 남은 문제(수치 그대로): 작은 목소리+마이크 잡음 51.4%(빈 결과 48/100), 잡음 5dB 33.9%(34/100), 잡음 10dB 18.9%(23/100). 이번에 본 방법(모델 7종, 문맥, 기준, 잡음 제거 3종)으로는 나아지지 않음. 다음 후보는 이런 조건의 실제 녹음으로 미세조정한 모델이나, 보내는 사람 쪽 디스코드 설정(입력 감도, 잡음 제거) 안내.

View File

@@ -68,6 +68,14 @@ TRACE_RESULT_CHARS = 2000
TRACE_THINKING_CHARS = 8000
def _cancelled(before: str) -> bool:
"""The answer is no longer wanted (turn_state.cancel): stop before the next model or tool call."""
if _turn_state.cancelled():
debug_log(f"reply cancelled before {before}", "planning")
return True
return False
def _clip(text: str, limit: int) -> str:
return text if len(text) <= limit else text[:limit] + "…"
@@ -1253,6 +1261,8 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
# Step 0.5: Deterministic on-screen site search. If the user named a site and
# asked to search/open it while broadcasting, do it directly — the small chat
# model otherwise just narrates without calling the browser tool.
if _cancelled("it started"):
return None
_site_search_reply = _maybe_deterministic_site_search(text, db, cfg, language)
if _site_search_reply is not None:
return _site_search_reply
@@ -1390,6 +1400,8 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
routed_tools = list(_cached_routed)
debug_log("tool router served from hot-window cache", "planning")
else:
if _cancelled("the tool router"):
return None
routed_tools = _route_tools(cfg, redacted, _builtin_tools, mcp_tools, recent_messages)
# Don't cache the router's "fall open to all tools" fallback. That
# path fires when the LLM router times out, returns empty, or emits
@@ -1532,6 +1544,8 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
if _one_step_tool:
debug_log(f"intent judge: one-step tool {_one_step_tool}, no plan", "planning")
elif not intent_judge or _turn_needs_tools:
if _cancelled("the planner"):
return None
action_plan = plan_query(
cfg=cfg,
query=redacted,
@@ -2376,8 +2390,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
_one_step_nudged = False
while turn < max_turns:
if _turn_state.cancelled():
debug_log("reply cancelled: no longer wanted", "planning")
if _cancelled("a model call"):
return None
turn += 1
debug_log(f"🔁 messages loop turn {turn}", "planning")
@@ -2496,6 +2509,8 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
}
],
})
if _cancelled("a planned tool call"):
return None
tools_started += 1
_plan_result = run_tool_with_retries(
db=db,
@@ -2844,8 +2859,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
continue
# Execute tool (not for an answer that is no longer wanted: no actions on its behalf)
if _turn_state.cancelled():
debug_log("reply cancelled before a tool call", "planning")
if _cancelled("a tool call"):
return None
tools_started += 1
result = run_tool_with_retries(

View File

@@ -32,7 +32,8 @@ class SttResult:
class Transcriber(Protocol):
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions) -> list[str]: ...
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions,
contexts: list[str] | None = None) -> list[str]: ...
Callback = Callable[[SttResult], None]
@@ -44,6 +45,7 @@ class _Job:
audio: np.ndarray
opts: DecodeOptions
callback: Callback
context: str = "" # finals only: the recent conversation, as the model's prompt
class SttScheduler:
@@ -74,8 +76,9 @@ class SttScheduler:
pass
self._executor.shutdown(wait=False, cancel_futures=True)
def submit_final(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback) -> None:
self._finals.append(_Job(key, audio, opts, callback))
def submit_final(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback,
context: str = "") -> None:
self._finals.append(_Job(key, audio, opts, callback, context))
self._wake.set()
def submit_partial(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback) -> None:
@@ -121,7 +124,7 @@ class SttScheduler:
t0 = time.perf_counter()
try:
texts = await loop.run_in_executor(self._executor, self.transcriber.transcribe, clips,
batch[0].opts)
batch[0].opts, [j.context for j in batch])
error = None
except Exception as e: # the model failing must not stop every speaker's STT
texts, error = [""] * len(batch), f"{type(e).__name__}: {e}"

View File

@@ -10,6 +10,7 @@ import itertools
import json
import secrets
import time
from collections import deque
from dataclasses import dataclass
from typing import Callable
@@ -28,6 +29,7 @@ from .tts import TtsEngine, TtsRequest, TtsService
CLOSE_BAD_HELLO = 4400
CLOSE_BAD_TOKEN = 4401
MAX_SPEAKERS = 32
CONTEXT_SECONDS = 180 # how far back the STT's conversation context reaches
def is_loopback(host: str) -> bool:
@@ -274,6 +276,9 @@ class Session:
self.chat_waiting: dict[str, asyncio.Future] = {}
self._chat_ids = itertools.count(1)
self.names: dict[str, str] = {}
# What was said and answered lately in Jarvis's voice channel: (time, text), the STT's
# conversation context (voice_worker.spec.md, "Conversation context"). Never logged.
self.lines: deque[tuple[float, str]] = deque(maxlen=50)
self.speakers: dict[str, _Speaker] = {}
self.tts_requests: dict[str, TtsRequest] = {}
# With recordings on: each spoken answer's text and mono samples, kept until it is made and the
@@ -364,6 +369,15 @@ class Session:
samples = st.converter.push(pcm)
self._handle_events(speaker, st, st.segmenter.push(samples, time.monotonic()))
def _context(self) -> str:
"""The recent conversation for a final's prompt: the channel's lines of the last
CONTEXT_SECONDS, at most `context_chars` characters from the end."""
n = self.settings.context_chars
if not n or not self.voice_channel:
return ""
since = time.monotonic() - CONTEXT_SECONDS
return " ".join(t for at, t in self.lines if at >= since)[-n:].strip()
def _opts(self, beam: int) -> DecodeOptions:
return DecodeOptions(beam=beam, language=self.settings.language, hotwords=self.settings.hotwords)
@@ -388,7 +402,8 @@ class Session:
else:
sched.submit_final((self.sid, speaker), loudness_equalise(ev.audio, ev.voiced),
self._opts(self.settings.final_beam),
lambda res, ev=ev, said=said: self._send_final(speaker, ev, res, said))
lambda res, ev=ev, said=said: self._send_final(speaker, ev, res, said),
context=self._context())
def _partial_cb(self, speaker: str, st: _Speaker, utt: int):
def cb(res: SttResult) -> None:
@@ -402,6 +417,8 @@ class Session:
store = self.worker.history
if store is not None and channel and res.text.strip():
store.add_voice(channel, speaker, name, res.text, ts)
if channel and channel == self.voice_channel and res.text.strip():
self.lines.append((time.monotonic(), res.text.strip()))
if self.worker.recorder is not None:
self.worker.recorder.said(ts, speaker, name, channel, res.text, ev.audio, SR)
self.send({"type": "final", "speaker": speaker, "utt": ev.utt, "text": res.text,
@@ -499,6 +516,8 @@ class Session:
rec["text"] += text
while len(self.tts_recorded) > 50: # never reported as played: not kept forever
self.tts_recorded.pop(next(iter(self.tts_recorded)))
if self.voice_channel and text.strip():
self.lines.append((time.monotonic(), text.strip()))
tts.feed(req, text, more)
def _on_tts_cancel(self, msg: dict) -> None:
@@ -523,6 +542,7 @@ class Session:
if self.voice_channel:
debug_log(f"session {self.sid} left voice channel {self.voice_channel}", "voice_worker")
self.voice_channel = None
self.lines.clear()
def _on_voice_join(self, msg: dict) -> None:
got = self._strings(msg, "channel")

View File

@@ -13,6 +13,7 @@ class SttSettings:
final_beam: int = 2
language: str | None = "ko"
hotwords: tuple[str, ...] = ()
context_chars: int = 200
def update(self, changes: dict) -> None:
"""Apply ``changes``; raises ValueError and changes nothing if any value is bad."""
@@ -52,6 +53,8 @@ def _check(key: str, value):
if value is not None and (not isinstance(value, str) or not 2 <= len(value) <= 8):
raise ValueError("language must be a language code or null")
return value
if key == "context_chars":
return _int_in(key, value, 0, 500)
if key == "hotwords":
if (not isinstance(value, (list, tuple)) or len(value) > 50
or not all(isinstance(w, str) and 0 < len(w) <= 64 for w in value)):

View File

@@ -20,6 +20,12 @@ SR = 16000
# Whisper's no_speech_prob is not used: large-v3-turbo reports 0.0 for silence and noise alike.
MIN_MEAN_LOGPROB = -0.7
MAX_COMPRESSION_RATIO = 2.4 # repetition loops ("A B A B A B ...") compress far better than speech
# faster-whisper keeps at most this many previous-text tokens in a prompt (max_length // 2 - 1).
MAX_PREVIOUS_TOKENS = 223
# A plain decode at or above this, though below MIN_MEAN_LOGPROB, may be rescued by the context
# (choose_with_context). Measured 2026-11-19 on harsh copies of the user's speech: down to -0.8 no
# more noise came out as text (VAD-cut noise 0/5, raw noise 16/108 as without); -0.9 let 3 more through.
UNSURE_LOGPROB = -0.8
def _letters(text: str) -> str:
@@ -73,6 +79,33 @@ def choose_with_hint(hint_text: str, hint_lp: float, plain_text: str, plain_lp:
return plain_text
def choose_with_context(heard: str, prompted: str, hotwords: tuple[str, ...],
plain: tuple[str, float] | None = None) -> str:
"""Pick between the decode without the conversation context (``heard``, already through the hint
choice) and the one that had it as Whisper's previous-text prompt (``prompted``, through the same
choice, so it passed the gate itself).
The context fixes words of the topic the model misheard, but on noise or unsure audio it can write
words of the prompt, so the decode without it decides, as for the hint: the prompted text is used
only when it stays close to what was heard and holds every hotword exactly as often. When nothing
was heard, ``plain`` (the plain decode's text and mean log-probability) may still be just short of
the gate (at least UNSURE_LOGPROB): then the prompted text is used if it is that close to it.
"""
if not heard:
if not prompted or plain is None or plain[1] < UNSURE_LOGPROB \
or compression_ratio(plain[0]) > MAX_COMPRESSION_RATIO:
return ""
return prompted if choose_with_context(plain[0], prompted, hotwords) == prompted else ""
if not prompted:
return heard
h, p = _letters(heard), _letters(prompted)
if _edit_distance(p, h) > len(h) // 2:
return heard
if any(_mentions(prompted, w) != _mentions(heard, w) for w in hotwords):
return heard
return prompted
def add_cuda_dll_dirs() -> None:
"""On Windows ctranslate2 loads cuBLAS/cuDNN from the PATH; the pip wheels put them in nvidia/*/bin."""
if not hasattr(os, "add_dll_directory"):
@@ -104,8 +137,34 @@ class WhisperTranscriber:
def warm_up(self) -> None:
self.transcribe([np.zeros(SR, dtype=np.float32)], DecodeOptions(beam=1, language="en", hotwords=()))
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions) -> list[str]:
"""Clips arrive loudness-equalised by the session."""
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions,
contexts: list[str] | None = None) -> list[str]:
"""Clips arrive loudness-equalised by the session. ``contexts``: per clip, the recent
conversation to prompt with ("" for none); see ``choose_with_context``."""
enc, langs = self._encode(clips, opts)
plain = self._decode(enc, langs, opts.beam, None)
heard = self._choose(enc, langs, opts, None, plain)
prev = self._previous(langs, contexts)
if prev is None:
return heard
prompted = self._choose(enc, langs, opts, prev, self._decode(enc, langs, opts.beam, None, prev))
return [choose_with_context(h, p, opts.hotwords, pl) if c else h
for h, p, pl, c in zip(heard, prompted, plain, contexts)]
def _previous(self, langs: list[str], contexts: list[str] | None) -> list[list[int]] | None:
"""The contexts as previous-text tokens of one length for the whole batch: the decoder needs
the start token at the same position in every prompt. Each is cut from its end to the
shortest one (a channel's clips share most of their context); a clip without one borrows
another's, and its prompted result is not used."""
if not contexts or not any(contexts):
return None
toks = [self._tokenizer(lang).encode(" " + ctx.strip())[-MAX_PREVIOUS_TOKENS:] if ctx else None
for lang, ctx in zip(langs, contexts)]
n = min(len(t) for t in toks if t is not None)
filler = next(t for t in toks if t is not None)[-n:]
return [t[-n:] if t is not None else filler for t in toks]
def _encode(self, clips: list[np.ndarray], opts: DecodeOptions):
m = self.model
n_frames = m.feature_extractor.nb_max_frames
feats = []
@@ -118,16 +177,20 @@ class WhisperTranscriber:
langs = [opts.language] * len(clips)
else:
langs = [r[0][0][2:-2] for r in m.model.detect_language(enc)]
plain = self._decode(enc, langs, opts.beam, None)
return enc, langs
def _choose(self, enc, langs: list[str], opts: DecodeOptions, prev: list[list[int]] | None,
plain: list[tuple[str, float]]) -> list[str]:
if not opts.hotwords:
return [text if keep_transcript(text, lp) else "" for text, lp in plain]
hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords))
hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords), prev)
return [choose_with_hint(ht, hlp, pt, plp, opts.hotwords) for (ht, hlp), (pt, plp) in zip(hinted, plain)]
def _decode(self, enc, langs: list[str], beam: int, hotwords: str | None) -> list[tuple[str, float]]:
def _decode(self, enc, langs: list[str], beam: int, hotwords: str | None,
prev: list[list[int]] | None = None) -> list[tuple[str, float]]:
m = self.model
prompts = [m.get_prompt(self._tokenizer(lang), [], without_timestamps=True, hotwords=hotwords)
for lang in langs]
prompts = [m.get_prompt(self._tokenizer(lang), p, without_timestamps=True, hotwords=hotwords)
for lang, p in zip(langs, prev or [[]] * len(langs))]
results = m.model.generate(enc, prompts, beam_size=beam, max_length=224, suppress_blank=True,
suppress_tokens=[-1], return_scores=True)
# ``scores`` is the sequence log-probability divided by its length: the mean per token.

View File

@@ -80,6 +80,7 @@ STT settings (all optional in `hello` and `config`):
| `end_silence_ms` | 300 | Trailing VAD silence that ends an utterance. Range 100-2000. |
| `final_beam` | 2 | Beam size for finals (partials always use 1). Range 1-5. |
| `language` | `"ko"` | Whisper language code, or `null` to detect per utterance. |
| `context_chars` | 200 | Finals are also decoded with the recent conversation as Whisper's previous-text prompt, at most this many characters (Conversation context). `0` turns it off. Range 0-500. |
| `hotwords` | `[]` | Words (wake words, names) passed to Whisper as a hint. The hint alone made Whisper write the wake word on laughter, so with hotwords every clip is also decoded without them and that plain decode decides (see Hallucination gate). |
### Utterance rules
@@ -152,6 +153,46 @@ text is empty:
`no_speech_prob` is not used: large-v3-turbo reports 0.0 for silence and noise alike.
### Conversation context
People talk about one thing for a while, and the words of that topic ("디코", "화면 공유", "계산기")
were the ones misheard (live 2026-11-19): Whisper heard a casual, unclear "화면 공유" as "화면공이"
and "계산기" as "계산지", and five models (large-v3-turbo, large-v3 and three Korean fine-tunes) made
the same mistakes. Whisper was trained to continue from the text before a clip, so a final is also
decoded with the recent conversation as its previous-text prompt:
- The context is what was said (the session's non-empty finals, everyone's) and what Jarvis said
(the text of every `tts` request) in the voice channel Jarvis is in, over the last 180 s, joined
by spaces, at most `context_chars` characters from the end. Joining another channel or leaving
clears it. It lives in memory only, is never logged and goes nowhere but the model's prompt.
- Partials have none (they stay one cheap decode).
- The clip is decoded as before (with the hotword choice above) and again with the context (the
same choice, each decode prompted). The prompted text is used only when the one without context
was kept, it stays within half that text's letters of it, and it holds every hotword exactly as
often; otherwise the text without context is used. On noise or unsure audio the prompt makes
Whisper write words of the conversation; this rule keeps that out, as for the hint.
- One rescue: when the decode without context was dropped by the gate but its mean log-probability
is at least -0.8 (just short of -0.7), the prompted text (which passed the gate itself) is used if
it is that close to it and keeps every hotword count. The unsure text itself is never used.
Measured on harsh copies of the user's speech: a cheap headset 13.8% to 7.5% error, empty results
fewer by 1-2 per 100 in most conditions, and no more noise with text (VAD-cut noise 0 of 5, noise
decoded straight 16 of 108, as before); -0.9 let 3 more noise clips through.
- Finals stay batched. The decoder needs the start token at the same position in every prompt of a
batch, so each clip's context is cut from its end to the shortest one in the batch (a channel's
clips share most of it); a clip without a context borrows another's and keeps its plain result.
Measured (2026-11-19, the game PC's RTX 5070 Ti with Strata loaded, large-v3-turbo int8_float16,
beam 2, the 100 recordings of the user's speech from 2026-11-16 to 19, 16 of them with the user's own
text): character error rate 12.0% without and 7.5% with the context (wrong sentences 10 to 8), no more
empty results (17 of 100 both ways), final decode 134 ms to 182 ms on average (p95 170 to 253 ms).
On the 108 noise clips decoded straight (no VAD), as many came out with text with the context as
without it (16 each); without the rule above it was 22 with turbo, and with large-v3 63, which wrote
whole sentences of the prompt. Under harsh copies of the same recordings (a weak voice over a mic
noise floor, ESC-50 noise at SNR 10 and 5 dB, a 300-3400 Hz headset, Opus 12 kbps, reverb) the
context never made the error rate worse (it fell most with Opus and reverb). Ten finals at once stay
one batch: 1.3 s with contexts, 1.6-2.0 s without (Strata running). Evidence and the model comparison:
`docs/v2/stt-accuracy-plan.md`.
Measured (2026-11-11, RTX 5050) with 54 real noise recordings from ESC-50 (typing, mouse, breathing,
coughing, laughing, crying, vacuum, ...), each at its own level and at -25 dBFS: of the 108 noise-only
clips streamed through the worker with the wake word hint on, 103 never reached the model (the VAD

View File

@@ -64,3 +64,74 @@ def test_an_answer_cancelled_during_a_model_call_runs_no_tool_for_it(mock_config
# The model asks for a tool, but the answer was taken back meanwhile: the tool is not run.
reply, calls, invoked = _run(mock_config, db, dialogue_memory, cancel_after_calls=1)
assert reply is None and calls == 1 and invoked == []
@pytest.mark.unit
def test_a_planned_tool_is_not_run_when_the_answer_is_cancelled_while_its_step_is_resolved(
mock_config, db, dialogue_memory):
# Small models run planned steps directly: the step resolver (a model call) returns a tool call,
# but the answer was taken back meanwhile.
from jarvis.reply import engine as engine_mod
from jarvis.reply import turn_state
mock_config.ollama_chat_model = "gemma4:e2b"
mock_config.evaluator_enabled = False
cancelled = [False]
invoked = []
def fake_resolve(*args, **kwargs):
cancelled[0] = True
return ("webSearch", {"query": "x"})
plan = ["webSearch query='a'", "webSearch query='b from step 1'", "Reply to the user."]
turn_state.reset()
turn_state.set_cancel(lambda: cancelled[0])
try:
with patch.object(engine_mod, "run_tool_with_retries", side_effect=lambda *a, **k: invoked.append(1)), \
patch.object(engine_mod, "chat_with_messages", side_effect=AssertionError("no model call")), \
patch.object(engine_mod, "select_tools", return_value=["webSearch", "stop"]), \
patch.object(engine_mod, "extract_search_params_for_memory", return_value={"keywords": []}), \
patch.object(engine_mod, "plan_query", return_value=plan), \
patch.object(engine_mod, "_resolve_plan_step", side_effect=fake_resolve):
reply = engine_mod.run_reply_engine(db=db, cfg=mock_config, tts=None, text="a then b",
dialogue_memory=dialogue_memory)
finally:
turn_state.reset()
assert reply is None and invoked == []
@pytest.mark.unit
@pytest.mark.parametrize("cancel_in", ["select_tools", "plan_query"])
def test_a_cancel_during_routing_or_planning_stops_before_the_next_call(mock_config, db, dialogue_memory,
cancel_in):
from jarvis.reply import engine as engine_mod
from jarvis.reply import turn_state
mock_config.ollama_chat_model = "gpt-oss:20b"
mock_config.evaluator_enabled = False
cancelled = [False]
calls = []
def route(*a, **k):
calls.append("route")
cancelled[0] = cancel_in == "select_tools"
return ["webSearch", "stop"]
def plan(*a, **k):
calls.append("plan")
cancelled[0] = True
return ["webSearch query='a'", "Reply to the user."]
turn_state.reset()
turn_state.set_cancel(lambda: cancelled[0])
try:
with patch.object(engine_mod, "chat_with_messages", side_effect=lambda *a, **k: calls.append("chat")), \
patch.object(engine_mod, "run_tool_with_retries", side_effect=lambda *a, **k: calls.append("tool")), \
patch.object(engine_mod, "select_tools", side_effect=route), \
patch.object(engine_mod, "plan_query", side_effect=plan):
reply = engine_mod.run_reply_engine(db=db, cfg=mock_config, tts=None, text="날씨 찾아 줘",
dialogue_memory=dialogue_memory)
finally:
turn_state.reset()
assert reply is None
assert calls == (["route"] if cancel_in == "select_tools" else ["route", "plan"])

View File

@@ -303,7 +303,7 @@ class SlowTranscriber(LevelTranscriber):
super().__init__()
self.delay = delay
def transcribe(self, clips, opts):
def transcribe(self, clips, opts, contexts=None):
import time as _t
_t.sleep(self.delay)
return super().transcribe(clips, opts)

View File

@@ -96,3 +96,68 @@ def test_wake_word_matching_ignores_case_spacing_and_punctuation():
def test_spacing_and_punctuation_differences_do_not_change_the_choice():
assert choose_with_hint("자비스 오늘 서울 날씨 어때", -0.1, "자비스, 오늘 서울 날씨 어때?", -0.1, HINT) \
== "자비스 오늘 서울 날씨 어때"
# The recent conversation as Whisper's previous-text prompt (voice_worker.spec.md, "Conversation
# context"). Live 2026-11-19 the prompt fixed words of the topic ("화면공이" -> "화면 공유", "계산지" ->
# "계산기"); on noise it wrote words of the prompt. The decode without it decides, as with the hint.
from voice_worker.stt_whisper import choose_with_context # noqa: E402
@pytest.mark.unit
def test_the_context_may_fix_words_of_what_was_said():
assert choose_with_context("내 화면공이 안 보여?", "내 화면 공유 안 보여?", HINT) == "내 화면 공유 안 보여?"
@pytest.mark.unit
def test_nothing_heard_without_the_context_stays_nothing():
# Noise: the prompted decode wrote words of the conversation.
assert choose_with_context("", "화면에 디스코드", HINT) == ""
@pytest.mark.unit
def test_a_prompted_decode_far_from_what_was_heard_is_not_used():
heard = "가상화면을 못 봐"
copied = "네, 현재 화면에 디스코드 채팅 창과 여러 프로그램이 정상적으로 보입니다."
assert choose_with_context(heard, copied, HINT) == heard
@pytest.mark.unit
def test_the_context_never_changes_how_many_wake_words_were_heard():
assert choose_with_context("라비스 지금 영상 재생되고 있어?", "자비스 지금 영상 재생되고 있어?", HINT) \
== "라비스 지금 영상 재생되고 있어?"
assert choose_with_context("자비스 날씨 알려 줘", "날씨 알려 줘", HINT) == "자비스 날씨 알려 줘"
@pytest.mark.unit
def test_an_empty_prompted_decode_keeps_what_was_heard():
assert choose_with_context("네.", "", HINT) == "네."
# A plain decode just short of the gate (live 2026-11-19 on a headset copy: "니코 어디갔어" at -0.72 for
# "디코 어디 갔어") is rescued by a confident prompted decode close to it; nothing else changes.
from voice_worker.stt_whisper import UNSURE_LOGPROB # noqa: E402
@pytest.mark.unit
def test_a_nearly_sure_decode_is_rescued_by_a_confident_close_prompted_one():
assert choose_with_context("", "디코 어디 갔어?", HINT, ("니코 어디갔어", MIN_MEAN_LOGPROB - 0.02)) \
== "디코 어디 갔어?"
@pytest.mark.unit
def test_an_unsure_decode_well_below_the_gate_is_not_rescued():
assert choose_with_context("", "디코 어디 갔어?", HINT, ("니코 어디갔어", UNSURE_LOGPROB - 0.05)) == ""
@pytest.mark.unit
def test_noise_with_a_prompt_far_from_it_is_not_rescued():
# The prompted decode wrote words of the conversation over laughter.
assert choose_with_context("", "화면에 디스코드", HINT, ("감사합니다.", MIN_MEAN_LOGPROB - 0.05)) == ""
assert choose_with_context("", "", HINT, ("감사합니다.", MIN_MEAN_LOGPROB - 0.05)) == ""
@pytest.mark.unit
def test_a_rescue_never_adds_a_wake_word():
assert choose_with_context("", "자비스 지금 영상 재생되고 있어?", HINT,
("라비스 지금 영상 재생되고 있어", MIN_MEAN_LOGPROB - 0.05)) == ""

View File

@@ -25,7 +25,7 @@ class FakeTranscriber:
self.gate.set()
self.fail_on_beam = fail_on_beam
def transcribe(self, clips, opts):
def transcribe(self, clips, opts, contexts=None):
self.entered.set()
self.gate.wait(5)
self.calls.append((opts, [len(c) for c in clips]))
@@ -218,12 +218,45 @@ class StepTranscriber:
def step(self):
self._steps.release()
def transcribe(self, clips, opts):
def transcribe(self, clips, opts, contexts=None):
self.calls.append([len(c) for c in clips])
self._steps.acquire(timeout=5)
return ["" for _ in clips]
@pytest.mark.unit
def test_each_final_reaches_the_model_with_its_own_context():
class Ctx:
def __init__(self):
self.calls = []
def transcribe(self, clips, opts, contexts=None):
self.calls.append(list(contexts or []))
return [f"n{len(c)}" for c in clips]
eng = Ctx()
async def body():
s = SttScheduler(eng)
s.start()
got = []
done = asyncio.Event()
def cb(r):
got.append(r.text)
if len(got) == 3:
done.set()
s.submit_final("a", clip(100), FINAL, cb, context="화면 공유")
s.submit_final("b", clip(200), FINAL, cb)
s.submit_partial("c", clip(300), PARTIAL, cb)
await asyncio.wait_for(done.wait(), 5)
await s.stop()
asyncio.run(body())
assert eng.calls[0] == ["화면 공유", ""]
assert eng.calls[1] == [""], "partials go without context"
@pytest.mark.unit
def test_partial_batches_are_kept_small_so_finals_can_cut_in():
t = StepTranscriber()

View File

@@ -32,7 +32,7 @@ class LevelTranscriber:
def __init__(self):
self.calls = []
def transcribe(self, clips, opts):
def transcribe(self, clips, opts, contexts=None):
self.calls.append((opts, len(clips)))
return [f"level {np.argmax(np.abs(np.fft.rfft(c))) * 16000 / len(c) / 1000:.1f}" for c in clips]
@@ -336,7 +336,7 @@ def test_48k_stereo_input_reaches_the_model_at_16k():
lengths = []
orig = t.transcribe
def spy(clips, opts):
def spy(clips, opts, contexts=None):
lengths.extend(len(c) for c in clips)
return orig(clips, opts)
@@ -496,3 +496,47 @@ def test_old_recordings_are_also_removed_while_the_worker_keeps_running(tmp_path
r.said(later.timestamp(), "1", "A", "vc", "나중", np.zeros(1600))
r.close()
assert sorted(p.name for p in tmp_path.iterdir()) == [later.date().isoformat()]
class ContextTranscriber(LevelTranscriber):
def __init__(self):
super().__init__()
self.contexts = []
def transcribe(self, clips, opts, contexts=None):
if opts.beam > 1: # finals
self.contexts += list(contexts or [""] * len(clips))
return super().transcribe(clips, opts)
async def _said(ws, speaker, level):
await stream(ws, speaker, level, 600)
await ws.send(json.dumps({"type": "speaking_end", "speaker": speaker}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final" and m["speaker"] == speaker)
@pytest.mark.unit
def test_each_final_is_transcribed_with_the_recent_conversation_of_the_channel():
"""voice_worker.spec.md, "Conversation context": what was said and answered just before, in the
channel Jarvis is in, is the model's prompt for the next final."""
t = ContextTranscriber()
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await ws.send(json.dumps({"type": "voice_join", "channel": "vc1"}))
await _said(ws, "100", 0.9)
await ws.send(json.dumps({"type": "tts", "id": "r1", "text": "디스코드 화면 공유가 보입니다.",
"more": False}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
await _said(ws, "200", 0.6)
await ws.send(json.dumps({"type": "voice_join", "channel": "vc2"})) # another channel
await _said(ws, "100", 0.9)
await ws.send(json.dumps({"type": "config", "context_chars": 0}))
await _said(ws, "100", 0.6)
asyncio.run(with_worker(body, transcriber=t, tts=FakeTts()))
assert t.contexts[0] == ""
assert t.contexts[1] == "level 0.9 디스코드 화면 공유가 보입니다."
assert t.contexts[2] == "", "a new channel starts without the old channel's conversation"
assert t.contexts[3] == "", "context_chars 0 turns it off"