Compare commits
3 Commits
app-v0.1.4
...
app-v0.1.4
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
09bdf05762 | ||
|
|
251f5e116d | ||
|
|
c1b561d66d |
2
app/Cargo.lock
generated
2
app/Cargo.lock
generated
@@ -1520,7 +1520,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.48"
|
||||
version = "0.1.49"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.48"
|
||||
version = "0.1.49"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"
|
||||
|
||||
4
app/shell/Cargo.lock
generated
4
app/shell/Cargo.lock
generated
@@ -2637,7 +2637,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.48"
|
||||
version = "0.1.49"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -2666,7 +2666,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.48"
|
||||
version = "0.1.49"
|
||||
dependencies = [
|
||||
"jarvis-app",
|
||||
"serde_json",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.48"
|
||||
version = "0.1.49"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"
|
||||
|
||||
@@ -179,6 +179,11 @@ struct Turn {
|
||||
ended_at: Instant,
|
||||
/// Still a "was it said to Jarvis?" question (its ask id), not yet a request of the policy.
|
||||
ask: Option<String>,
|
||||
/// The speaker started their next utterance within `call.continueMs` while the answer was not
|
||||
/// heard: the request stays open for that utterance's final, also if the answer starts meanwhile.
|
||||
held: bool,
|
||||
/// Its answer started playing (kept only while `held`).
|
||||
heard: bool,
|
||||
}
|
||||
|
||||
impl<V: Voice> Conversation<V> {
|
||||
@@ -274,6 +279,10 @@ impl<V: Voice> Conversation<V> {
|
||||
if let Some((_, deadline)) = self.wakes.get_mut(&h.speaker_id) {
|
||||
*deadline = None; // they go on: wait for this utterance's final
|
||||
}
|
||||
let window = self.calls.continue_window;
|
||||
if let Some(t) = self.turns.get_mut(&h.speaker_id) {
|
||||
t.held = !window.is_zero() && now <= t.ended_at + window;
|
||||
}
|
||||
}
|
||||
CoreEvent::Chat(m) => self.on_chat(m, now),
|
||||
CoreEvent::Partial(h, partial) => self.on_partial(h.speaker_id, partial),
|
||||
@@ -352,7 +361,7 @@ impl<V: Voice> Conversation<V> {
|
||||
let id = format!("q{}", req.id);
|
||||
if self.voice.ask_addressed(&id, speaker, &text, room) {
|
||||
self.turns.insert(speaker, Turn { id: req.id, text: req.text.clone(),
|
||||
ended_at: req.ended_at, ask: Some(id.clone()) });
|
||||
ended_at: req.ended_at, ask: Some(id.clone()), held: false, heard: false });
|
||||
self.asks.insert(id, (req, now));
|
||||
} else {
|
||||
let a = self.policy.utterance_closed(speaker, now);
|
||||
@@ -384,7 +393,10 @@ impl<V: Voice> Conversation<V> {
|
||||
// The answer is heard: what its asker says next is a new request.
|
||||
CoreEvent::SpeakStart(id) => {
|
||||
if let Some(pid) = self.by_speech.get(id).copied() {
|
||||
self.turns.retain(|_, t| t.id != pid);
|
||||
self.turns.retain(|_, t| t.id != pid || t.held);
|
||||
for t in self.turns.values_mut().filter(|t| t.id == pid) {
|
||||
t.heard = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
CoreEvent::SpeakEnd(e) => {
|
||||
@@ -419,7 +431,8 @@ impl<V: Voice> Conversation<V> {
|
||||
|
||||
/// A voice request goes to the policy; until its answer is heard, its speaker may go on.
|
||||
fn voice_request(&mut self, req: Request, now: Instant) {
|
||||
self.turns.insert(req.speaker, Turn { id: req.id, text: req.text.clone(), ended_at: req.ended_at, ask: None });
|
||||
self.turns.insert(req.speaker, Turn { id: req.id, text: req.text.clone(), ended_at: req.ended_at, ask: None,
|
||||
held: false, heard: false });
|
||||
let a = self.policy.request(req, now);
|
||||
self.apply(a, now);
|
||||
}
|
||||
@@ -428,10 +441,16 @@ impl<V: Voice> Conversation<V> {
|
||||
/// `call.continueMs` of that request's end, and is not a stop): the request is taken back, its
|
||||
/// running answer stopped. Returns its text and whether its earcon already played.
|
||||
fn going_on(&mut self, speaker: u64, f: &crate::discord::HeardFinal, now: Instant) -> Option<(String, bool)> {
|
||||
let t = self.turns.get(&speaker)?;
|
||||
let t = self.turns.get_mut(&speaker)?;
|
||||
let started = now.checked_sub(Duration::from_millis(f.wait_ms + f.audio_ms)).unwrap_or(now);
|
||||
if self.calls.continue_window.is_zero() || started > t.ended_at + self.calls.continue_window
|
||||
// An empty final (noise the worker rejected) adds nothing and asks nothing again.
|
||||
if crate::calls::normalise(&f.text).is_empty() || self.calls.continue_window.is_zero()
|
||||
|| started > t.ended_at + self.calls.continue_window
|
||||
|| matches!(self.calls.judge(speaker, &f.text), Call::Stop) {
|
||||
t.held = false;
|
||||
if t.heard {
|
||||
self.turns.remove(&speaker);
|
||||
}
|
||||
return None;
|
||||
}
|
||||
let t = self.turns.remove(&speaker).expect("checked");
|
||||
@@ -1156,6 +1175,33 @@ mod tests {
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "자비스 날씨 알려 줘".into()), Did::StopAll, Did::Cancel("s1".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_that_started_before_the_answer_played_is_still_joined() {
|
||||
// The next utterance starts while Jarvis thinks; the answer starts playing before its final.
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "지금 화면 봐봐", 100), ms(t, 0));
|
||||
c.on_event(&start(A, 2), ms(t, 1000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 1500));
|
||||
c.on_event(&fin_utt(A, 2, "디코 화면 공유 창에 가려져 있잖아"), ms(t, 3000));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "지금 화면 봐봐".into()), Did::Cancel("s1".into()),
|
||||
Did::Answer(A, "지금 화면 봐봐 디코 화면 공유 창에 가려져 있잖아".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_final_leaves_the_request_as_it_is() {
|
||||
// Noise the worker rejected comes as an empty final: nothing to join, nothing asked again.
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
c.on_event(&fin(A, "계산기 켜 줘", 100), ms(t, 0));
|
||||
c.on_event(&fin(A, "", 100), ms(t, 1500));
|
||||
assert_eq!(f.take(), vec![Did::Answer(A, "계산기 켜 줘".into())]);
|
||||
// An empty final after the answer played ends the hold: what follows is a new request.
|
||||
c.on_event(&start(A, 2), ms(t, 2000));
|
||||
c.on_event(&speak_start("s1"), ms(t, 2500));
|
||||
c.on_event(&fin_utt(A, 2, " "), ms(t, 3200));
|
||||
c.on_event(&fin(A, "고마워", 100), ms(t, 4500));
|
||||
assert_eq!(f.take(), vec![Did::Cancel("s1".into()), Did::Answer(A, "고마워".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn going_on_lets_the_brain_get_ready_for_the_joined_request() {
|
||||
let (mut c, f, t) = alone(PolicyKind::Latest);
|
||||
|
||||
@@ -93,7 +93,11 @@ started within `call.continueMs` of the request's end (start = the final's arriv
|
||||
- The joined text (the earlier text, a space, the new final) is judged and answered as one request,
|
||||
as if it had been said in one go; it ends when the new final ended. It may be joined again.
|
||||
- The new final is joined whatever it is on its own (a piece needs no wake word of its own), except
|
||||
a stop: a stop stops, as always.
|
||||
a stop (a stop stops, as always) and an empty final (noise the worker rejected: nothing is joined
|
||||
or asked again, the request goes on as it was).
|
||||
- When the speaker's next utterance starts (`SpeechStart`) within `call.continueMs` of the request's
|
||||
end, the request is held open for that utterance's final, also if its answer starts playing
|
||||
meanwhile; the joined request then stops that answer. An empty final ends the hold.
|
||||
- Other speakers' utterances never join it. An answer already heard, an answer that ended, a missed
|
||||
request, a stop, leaving and a lost worker end it; then what the speaker says is a new request.
|
||||
- The joined request gets no second earcon when the first one's already played.
|
||||
@@ -224,7 +228,8 @@ one request, stop all, earcon, prepare, ask "addressed", ask "rank"), so the glu
|
||||
cannot be queued the answer is done at once); `StopAnswer` → `cancel` of that answer's speech;
|
||||
`Rank { id }` → `rank` with id `k<id>` and the candidates' request ids; `Missed` → a debug log and
|
||||
the missed list (the last 100: speaker, reason, time and length, never the text).
|
||||
- `SpeakStart` of an answer's speech: the answer is heard, its request is no longer open (Going on).
|
||||
- `SpeakStart` of an answer's speech: the answer is heard, its request is no longer open (Going on),
|
||||
unless its speaker already started their next utterance (held).
|
||||
- `SpeakEnd` of an answer's speech is that answer's done. The end of a speech the loop already
|
||||
cancelled starts nothing.
|
||||
- `Stopped` (all speech stopped from elsewhere: the slash command, `stop()`, the session ending) and
|
||||
@@ -248,7 +253,8 @@ off, partials that read as a call sent to get ready (joined to a waiting wake wo
|
||||
answering someone else, the joined text while going on), going on (pieces joined into one answer in
|
||||
every policy without a missed record, a piece without a wake word joined, other speakers never, a
|
||||
new request once the answer is heard or the pause is longer than `call.continueMs`, a stop still
|
||||
stopping, no second earcon), interruption cancelling the right speech, a ranking deciding, end times from
|
||||
stopping, no second earcon, an utterance started before the answer played still joined, an empty
|
||||
final joining nothing), interruption cancelling the right speech, a ranking deciding, end times from
|
||||
`wait_ms`, and chat: an answer posted as a reply once, per-channel switches, no answers to bots or to
|
||||
a stop, chat wake words and designated people, a failed answer posting nothing). `tests/worker_protocol.rs` checks an answer spoken under its id and both judgements against
|
||||
the real Python worker with a fake brain.
|
||||
|
||||
@@ -85,6 +85,14 @@
|
||||
- 2026-11-19 저녁 (게임PC, 사용자가 "지금 컴퓨터 사용해도 됨"): 사용자가 내보낸 녹음과 대화 기록을 받음. 사용자 녹음 100개(11-16 11개, 11-18 53개, 11-19 36개), 정답 16개(11-16 시험 문장 11개 + 사용자가 준 5개).
|
||||
- 불만 2(끊어 말하기)의 원인은 STT가 아니었음: 한 생각 안의 쉼이 0.1~4.7초라 조각마다 따로 요청이 되었고, 앱이 새 조각으로 앞 답을 멈춰도 워커는 그 답을 끝까지 만들어 다음 답이 13~35초 기다림. 0.1.47에서 고침(conversation.spec.md "Going on", voice_worker.spec.md: 취소된 답은 시작하지 않거나 다음 모델·도구 호출 전에 멈춤).
|
||||
- H2: 녹음을 다시 봐도 소리 손상은 없었음(채널 64kbps, 4~7kHz 성분 정상). 빈 결과 17개는 모델이 확신하지 못해 버린 것(평균 확신도 -0.7 미만, "감사합니다" 같은 지어낸 말 포함).
|
||||
- H3 모델 비교(게임PC GPU, 워커와 같은 경로, int8_float16, 정답 16개 기준 문자 오류율): large-v3-turbo 12.0%, large-v3 12.7%, BuzzASR/korean 13.3%, batisay-ko-turbo 11.2%, ghost613 turbo-korean 33.1%. 모두 같은 곳을 틀림("기사야", "화면군이", "기구", "화면" 빠짐). 상대 20% 넘게 나은 모델이 없어서 모델은 바꾸지 않음. Qwen3-ASR은 VRAM(Strata와 함께 남는 약 2.5GB)에 들어가지 않아 서버 CPU에서 따로 봄.
|
||||
- H3 모델 비교(게임PC GPU, 워커와 같은 경로, int8_float16, 정답 16개 기준 문자 오류율): large-v3-turbo 12.0%, large-v3 12.7%, BuzzASR/korean 13.3%, batisay-ko-turbo 11.2%, ghost613 turbo-korean 33.1%. 모두 같은 곳을 틀림("기사야", "화면군이", "기구", "화면" 빠짐). 상대 20% 넘게 나은 모델이 없어서 모델은 바꾸지 않음. Qwen3-ASR-1.7B(Apache-2.0)는 서버 CPU에서 정답 16개만: 9.4%(한국어판 TeamUNIVA beta 11.7%), 같은 곳을 틀림("기사야", "공리", "가면", "기고"). 앞 대화 문맥을 준 turbo(7.5%)보다 못하고 bf16로 약 5GB가 필요해 Strata와 함께 남는 VRAM(약 2.5GB)에 들어가지 않아 바꾸지 않음.
|
||||
- H4 앞 대화 문맥: 12.0% → 7.5%(상대 38%), 빈 결과 그대로, 잡음 108개의 지어낸 말 그대로(16개), 최종 전사 평균 182ms(p95 253ms). 문맥만 주면 잡음에서 앞 대화 낱말을 씀(turbo 22개, large-v3 63개)이라 호출어 힌트와 같은 규칙으로 막음. 0.1.48에 넣음(voice_worker.spec.md "Conversation context").
|
||||
- 남은 것: 첫 낱말(호출어)과 짧은 말("기사야"→비서야, "기구"→디코)은 앞 대화가 없거나 너무 짧아 그대로. 호출 판정의 비슷한 말 허용이 호출어 쪽을 덮음.
|
||||
- 2026-11-19 밤 (검토 뒤): 0.1.49에서 고침
|
||||
- 이어 말하기: 다음 말이 시작된 뒤 답이 나오기 시작해도 그 말의 최종 결과까지 앞 요청을 붙잡아 둠. 빈 결과(잡음)는 합치지 않고 앞 요청을 다시 묻지 않음.
|
||||
- 취소: 답 시작, 도구 고르기, 계획, 계획한 도구 실행 직전에도 확인.
|
||||
- 문맥을 쓸 때도 여러 사람의 최종 전사를 한 번에 묶음(문맥을 묶음 안에서 가장 짧은 길이로 끝에서 맞춤). 10명 동시: 묶음 1.3초(문맥 없이 1.6~2.0초, Strata가 돌던 게임PC), VRAM 최대 15.7GB/16.3GB(시작 14.6GB).
|
||||
- B. 악조건(사용자 녹음 100개, 정답 16개, 문맥 없음 → 있음 문자 오류율): 원본 11.9→7.0%, 작은 목소리+마이크 잡음 52.8→51.4%, 잡음 SNR 10dB 18.8→18.9%, 5dB 34.2→33.9%, 싼 헤드셋 14.9→13.8%, Opus 12kbps 12.8→9.8%, 방 울림 13.0→8.8%. 문맥은 어느 조건에서도 나쁘게 하지 않음. 하지만 악조건 자체가 나쁨: 빈 결과가 17개에서 20~49개로 늘어남.
|
||||
- 빈 결과의 원인은 확신도 기준(-0.7). -0.85/-1.0으로 낮추면 헤드셋 7.5%, 잡음 10dB 15.5%, 5dB 30.2%, 빈 결과 원본 17→15/10개. 대신 잡음 108개(VAD 없이 바로 넣음)에서 글자가 나온 것이 16→27/32개. 실제 워커는 VAD가 먼저 거르므로(M1: 108개 중 5개만 모델까지 감) 기준 변경은 VAD를 포함한 잡음 시험으로 다시 재고 사용자가 정할 것. 이번에는 바꾸지 않음.
|
||||
- 작은 목소리+마이크 잡음(약 SNR 5dB)은 어떤 설정으로도 50% 안팎이라, 소리 앞단의 잡음 제거(모델) 같은 다른 방향이 필요함. 아직 미검증.
|
||||
- 근거: groups/javis_bot/evidence/2026-11-19/stt-plan/harsh/ (build.py 조건 정의, harsh.py, gate.py, 결과 JSON).
|
||||
|
||||
@@ -68,6 +68,14 @@ TRACE_RESULT_CHARS = 2000
|
||||
TRACE_THINKING_CHARS = 8000
|
||||
|
||||
|
||||
def _cancelled(before: str) -> bool:
|
||||
"""The answer is no longer wanted (turn_state.cancel): stop before the next model or tool call."""
|
||||
if _turn_state.cancelled():
|
||||
debug_log(f"reply cancelled before {before}", "planning")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _clip(text: str, limit: int) -> str:
|
||||
return text if len(text) <= limit else text[:limit] + "…"
|
||||
|
||||
@@ -1253,6 +1261,8 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
# Step 0.5: Deterministic on-screen site search. If the user named a site and
|
||||
# asked to search/open it while broadcasting, do it directly — the small chat
|
||||
# model otherwise just narrates without calling the browser tool.
|
||||
if _cancelled("it started"):
|
||||
return None
|
||||
_site_search_reply = _maybe_deterministic_site_search(text, db, cfg, language)
|
||||
if _site_search_reply is not None:
|
||||
return _site_search_reply
|
||||
@@ -1390,6 +1400,8 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
routed_tools = list(_cached_routed)
|
||||
debug_log("tool router served from hot-window cache", "planning")
|
||||
else:
|
||||
if _cancelled("the tool router"):
|
||||
return None
|
||||
routed_tools = _route_tools(cfg, redacted, _builtin_tools, mcp_tools, recent_messages)
|
||||
# Don't cache the router's "fall open to all tools" fallback. That
|
||||
# path fires when the LLM router times out, returns empty, or emits
|
||||
@@ -1532,6 +1544,8 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
if _one_step_tool:
|
||||
debug_log(f"intent judge: one-step tool {_one_step_tool}, no plan", "planning")
|
||||
elif not intent_judge or _turn_needs_tools:
|
||||
if _cancelled("the planner"):
|
||||
return None
|
||||
action_plan = plan_query(
|
||||
cfg=cfg,
|
||||
query=redacted,
|
||||
@@ -2376,8 +2390,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
_one_step_nudged = False
|
||||
|
||||
while turn < max_turns:
|
||||
if _turn_state.cancelled():
|
||||
debug_log("reply cancelled: no longer wanted", "planning")
|
||||
if _cancelled("a model call"):
|
||||
return None
|
||||
turn += 1
|
||||
debug_log(f"🔁 messages loop turn {turn}", "planning")
|
||||
@@ -2496,6 +2509,8 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
}
|
||||
],
|
||||
})
|
||||
if _cancelled("a planned tool call"):
|
||||
return None
|
||||
tools_started += 1
|
||||
_plan_result = run_tool_with_retries(
|
||||
db=db,
|
||||
@@ -2844,8 +2859,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
continue
|
||||
|
||||
# Execute tool (not for an answer that is no longer wanted: no actions on its behalf)
|
||||
if _turn_state.cancelled():
|
||||
debug_log("reply cancelled before a tool call", "planning")
|
||||
if _cancelled("a tool call"):
|
||||
return None
|
||||
tools_started += 1
|
||||
result = run_tool_with_retries(
|
||||
|
||||
@@ -19,7 +19,9 @@ SR = 16000
|
||||
# over it: every hallucination stayed below, real speech above except two short words in loud noise.
|
||||
# Whisper's no_speech_prob is not used: large-v3-turbo reports 0.0 for silence and noise alike.
|
||||
MIN_MEAN_LOGPROB = -0.7
|
||||
MAX_COMPRESSION_RATIO = 2.4 # repetition loops ("A B A B A B ...") compress far better than speech
|
||||
MAX_COMPRESSION_RATIO = 2.4
|
||||
# faster-whisper keeps at most this many previous-text tokens in a prompt (max_length // 2 - 1).
|
||||
MAX_PREVIOUS_TOKENS = 223 # repetition loops ("A B A B A B ...") compress far better than speech
|
||||
|
||||
|
||||
def _letters(text: str) -> str:
|
||||
@@ -127,19 +129,27 @@ class WhisperTranscriber:
|
||||
contexts: list[str] | None = None) -> list[str]:
|
||||
"""Clips arrive loudness-equalised by the session. ``contexts``: per clip, the recent
|
||||
conversation to prompt with ("" for none); see ``choose_with_context``."""
|
||||
if contexts and any(contexts):
|
||||
# Prompts of different lengths cannot share one generate call (the start token must sit
|
||||
# at the same position): one clip at a time. The encoder costs the same per clip anyway.
|
||||
return [self._with_context(c, opts, ctx) for c, ctx in zip(clips, contexts)]
|
||||
enc, langs = self._encode(clips, opts)
|
||||
return self._choose(enc, langs, opts, None)
|
||||
|
||||
def _with_context(self, clip: np.ndarray, opts: DecodeOptions, context: str) -> str:
|
||||
enc, langs = self._encode([clip], opts)
|
||||
heard = self._choose(enc, langs, opts, None)[0]
|
||||
if not context:
|
||||
heard = self._choose(enc, langs, opts, None)
|
||||
prev = self._previous(langs, contexts)
|
||||
if prev is None:
|
||||
return heard
|
||||
return choose_with_context(heard, self._choose(enc, langs, opts, [context])[0], opts.hotwords)
|
||||
prompted = self._choose(enc, langs, opts, prev)
|
||||
return [choose_with_context(h, p, opts.hotwords) if c else h
|
||||
for h, p, c in zip(heard, prompted, contexts)]
|
||||
|
||||
def _previous(self, langs: list[str], contexts: list[str] | None) -> list[list[int]] | None:
|
||||
"""The contexts as previous-text tokens of one length for the whole batch: the decoder needs
|
||||
the start token at the same position in every prompt. Each is cut from its end to the
|
||||
shortest one (a channel's clips share most of their context); a clip without one borrows
|
||||
another's, and its prompted result is not used."""
|
||||
if not contexts or not any(contexts):
|
||||
return None
|
||||
toks = [self._tokenizer(lang).encode(" " + ctx.strip())[-MAX_PREVIOUS_TOKENS:] if ctx else None
|
||||
for lang, ctx in zip(langs, contexts)]
|
||||
n = min(len(t) for t in toks if t is not None)
|
||||
filler = next(t for t in toks if t is not None)[-n:]
|
||||
return [t[-n:] if t is not None else filler for t in toks]
|
||||
|
||||
def _encode(self, clips: list[np.ndarray], opts: DecodeOptions):
|
||||
m = self.model
|
||||
@@ -156,20 +166,18 @@ class WhisperTranscriber:
|
||||
langs = [r[0][0][2:-2] for r in m.model.detect_language(enc)]
|
||||
return enc, langs
|
||||
|
||||
def _choose(self, enc, langs: list[str], opts: DecodeOptions, contexts: list[str] | None) -> list[str]:
|
||||
plain = self._decode(enc, langs, opts.beam, None, contexts)
|
||||
def _choose(self, enc, langs: list[str], opts: DecodeOptions, prev: list[list[int]] | None) -> list[str]:
|
||||
plain = self._decode(enc, langs, opts.beam, None, prev)
|
||||
if not opts.hotwords:
|
||||
return [text if keep_transcript(text, lp) else "" for text, lp in plain]
|
||||
hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords), contexts)
|
||||
hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords), prev)
|
||||
return [choose_with_hint(ht, hlp, pt, plp, opts.hotwords) for (ht, hlp), (pt, plp) in zip(hinted, plain)]
|
||||
|
||||
def _decode(self, enc, langs: list[str], beam: int, hotwords: str | None,
|
||||
contexts: list[str] | None = None) -> list[tuple[str, float]]:
|
||||
prev: list[list[int]] | None = None) -> list[tuple[str, float]]:
|
||||
m = self.model
|
||||
prompts = [m.get_prompt(self._tokenizer(lang),
|
||||
self._tokenizer(lang).encode(" " + ctx.strip()) if ctx else [],
|
||||
without_timestamps=True, hotwords=hotwords)
|
||||
for lang, ctx in zip(langs, contexts or [""] * len(langs))]
|
||||
prompts = [m.get_prompt(self._tokenizer(lang), p, without_timestamps=True, hotwords=hotwords)
|
||||
for lang, p in zip(langs, prev or [[]] * len(langs))]
|
||||
results = m.model.generate(enc, prompts, beam_size=beam, max_length=224, suppress_blank=True,
|
||||
suppress_tokens=[-1], return_scores=True)
|
||||
# ``scores`` is the sequence log-probability divided by its length: the mean per token.
|
||||
|
||||
@@ -171,8 +171,9 @@ decoded with the recent conversation as its previous-text prompt:
|
||||
was kept, it stays within half that text's letters of it, and it holds every hotword exactly as
|
||||
often; otherwise the text without context is used. On noise or unsure audio the prompt makes
|
||||
Whisper write words of the conversation; this rule keeps that out, as for the hint.
|
||||
- Prompts of different lengths cannot share one decoder call, so finals with a context are decoded
|
||||
one clip at a time (the encoder costs the same per clip).
|
||||
- Finals stay batched. The decoder needs the start token at the same position in every prompt of a
|
||||
batch, so each clip's context is cut from its end to the shortest one in the batch (a channel's
|
||||
clips share most of it); a clip without a context borrows another's and keeps its plain result.
|
||||
|
||||
Measured (2026-11-19, the game PC's RTX 5070 Ti with Strata loaded, large-v3-turbo int8_float16,
|
||||
beam 2, the 100 recordings of the user's speech from 2026-11-16 to 19, 16 of them with the user's own
|
||||
@@ -180,7 +181,11 @@ text): character error rate 12.0% without and 7.5% with the context (wrong sente
|
||||
empty results (17 of 100 both ways), final decode 134 ms to 182 ms on average (p95 170 to 253 ms).
|
||||
On the 108 noise clips decoded straight (no VAD), as many came out with text with the context as
|
||||
without it (16 each); without the rule above it was 22 with turbo, and with large-v3 63, which wrote
|
||||
whole sentences of the prompt. Evidence and the model comparison: `docs/v2/stt-accuracy-plan.md`.
|
||||
whole sentences of the prompt. Under harsh copies of the same recordings (a weak voice over a mic
|
||||
noise floor, ESC-50 noise at SNR 10 and 5 dB, a 300-3400 Hz headset, Opus 12 kbps, reverb) the
|
||||
context never made the error rate worse (it fell most with Opus and reverb). Ten finals at once stay
|
||||
one batch: 1.3 s with contexts, 1.6-2.0 s without (Strata running). Evidence and the model comparison:
|
||||
`docs/v2/stt-accuracy-plan.md`.
|
||||
|
||||
Measured (2026-11-11, RTX 5050) with 54 real noise recordings from ESC-50 (typing, mouse, breathing,
|
||||
coughing, laughing, crying, vacuum, ...), each at its own level and at -25 dBFS: of the 108 noise-only
|
||||
|
||||
@@ -64,3 +64,74 @@ def test_an_answer_cancelled_during_a_model_call_runs_no_tool_for_it(mock_config
|
||||
# The model asks for a tool, but the answer was taken back meanwhile: the tool is not run.
|
||||
reply, calls, invoked = _run(mock_config, db, dialogue_memory, cancel_after_calls=1)
|
||||
assert reply is None and calls == 1 and invoked == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_planned_tool_is_not_run_when_the_answer_is_cancelled_while_its_step_is_resolved(
|
||||
mock_config, db, dialogue_memory):
|
||||
# Small models run planned steps directly: the step resolver (a model call) returns a tool call,
|
||||
# but the answer was taken back meanwhile.
|
||||
from jarvis.reply import engine as engine_mod
|
||||
from jarvis.reply import turn_state
|
||||
|
||||
mock_config.ollama_chat_model = "gemma4:e2b"
|
||||
mock_config.evaluator_enabled = False
|
||||
cancelled = [False]
|
||||
invoked = []
|
||||
|
||||
def fake_resolve(*args, **kwargs):
|
||||
cancelled[0] = True
|
||||
return ("webSearch", {"query": "x"})
|
||||
|
||||
plan = ["webSearch query='a'", "webSearch query='b from step 1'", "Reply to the user."]
|
||||
turn_state.reset()
|
||||
turn_state.set_cancel(lambda: cancelled[0])
|
||||
try:
|
||||
with patch.object(engine_mod, "run_tool_with_retries", side_effect=lambda *a, **k: invoked.append(1)), \
|
||||
patch.object(engine_mod, "chat_with_messages", side_effect=AssertionError("no model call")), \
|
||||
patch.object(engine_mod, "select_tools", return_value=["webSearch", "stop"]), \
|
||||
patch.object(engine_mod, "extract_search_params_for_memory", return_value={"keywords": []}), \
|
||||
patch.object(engine_mod, "plan_query", return_value=plan), \
|
||||
patch.object(engine_mod, "_resolve_plan_step", side_effect=fake_resolve):
|
||||
reply = engine_mod.run_reply_engine(db=db, cfg=mock_config, tts=None, text="a then b",
|
||||
dialogue_memory=dialogue_memory)
|
||||
finally:
|
||||
turn_state.reset()
|
||||
assert reply is None and invoked == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@pytest.mark.parametrize("cancel_in", ["select_tools", "plan_query"])
|
||||
def test_a_cancel_during_routing_or_planning_stops_before_the_next_call(mock_config, db, dialogue_memory,
|
||||
cancel_in):
|
||||
from jarvis.reply import engine as engine_mod
|
||||
from jarvis.reply import turn_state
|
||||
|
||||
mock_config.ollama_chat_model = "gpt-oss:20b"
|
||||
mock_config.evaluator_enabled = False
|
||||
cancelled = [False]
|
||||
calls = []
|
||||
|
||||
def route(*a, **k):
|
||||
calls.append("route")
|
||||
cancelled[0] = cancel_in == "select_tools"
|
||||
return ["webSearch", "stop"]
|
||||
|
||||
def plan(*a, **k):
|
||||
calls.append("plan")
|
||||
cancelled[0] = True
|
||||
return ["webSearch query='a'", "Reply to the user."]
|
||||
|
||||
turn_state.reset()
|
||||
turn_state.set_cancel(lambda: cancelled[0])
|
||||
try:
|
||||
with patch.object(engine_mod, "chat_with_messages", side_effect=lambda *a, **k: calls.append("chat")), \
|
||||
patch.object(engine_mod, "run_tool_with_retries", side_effect=lambda *a, **k: calls.append("tool")), \
|
||||
patch.object(engine_mod, "select_tools", side_effect=route), \
|
||||
patch.object(engine_mod, "plan_query", side_effect=plan):
|
||||
reply = engine_mod.run_reply_engine(db=db, cfg=mock_config, tts=None, text="날씨 찾아 줘",
|
||||
dialogue_memory=dialogue_memory)
|
||||
finally:
|
||||
turn_state.reset()
|
||||
assert reply is None
|
||||
assert calls == (["route"] if cancel_in == "select_tools" else ["route", "plan"])
|
||||
|
||||
Reference in New Issue
Block a user