Compare commits
2 Commits
app-v0.1.4
...
app-v0.1.4
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9a527aa470 | ||
|
|
628d89c4f1 |
2
app/Cargo.lock
generated
2
app/Cargo.lock
generated
@@ -1520,7 +1520,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.47"
|
||||
version = "0.1.48"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.47"
|
||||
version = "0.1.48"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"
|
||||
|
||||
4
app/shell/Cargo.lock
generated
4
app/shell/Cargo.lock
generated
@@ -2637,7 +2637,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.47"
|
||||
version = "0.1.48"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -2666,7 +2666,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.47"
|
||||
version = "0.1.48"
|
||||
dependencies = [
|
||||
"jarvis-app",
|
||||
"serde_json",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.47"
|
||||
version = "0.1.48"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"
|
||||
|
||||
@@ -82,3 +82,9 @@
|
||||
## 진행 기록
|
||||
|
||||
- 2026-11-19: 서버 GPU 사용 불가. 15:19 재부팅으로 커널 7.0.0-43이 올라왔는데 이 커널용 nvidia 모듈 패키지(`linux-modules-nvidia-595-open-7.0.0-43-generic`)가 설치되지 않음. 서버 측정은 CPU로만 가능.
|
||||
- 2026-11-19 저녁 (게임PC, 사용자가 "지금 컴퓨터 사용해도 됨"): 사용자가 내보낸 녹음과 대화 기록을 받음. 사용자 녹음 100개(11-16 11개, 11-18 53개, 11-19 36개), 정답 16개(11-16 시험 문장 11개 + 사용자가 준 5개).
|
||||
- 불만 2(끊어 말하기)의 원인은 STT가 아니었음: 한 생각 안의 쉼이 0.1~4.7초라 조각마다 따로 요청이 되었고, 앱이 새 조각으로 앞 답을 멈춰도 워커는 그 답을 끝까지 만들어 다음 답이 13~35초 기다림. 0.1.47에서 고침(conversation.spec.md "Going on", voice_worker.spec.md: 취소된 답은 시작하지 않거나 다음 모델·도구 호출 전에 멈춤).
|
||||
- H2: 녹음을 다시 봐도 소리 손상은 없었음(채널 64kbps, 4~7kHz 성분 정상). 빈 결과 17개는 모델이 확신하지 못해 버린 것(평균 확신도 -0.7 미만, "감사합니다" 같은 지어낸 말 포함).
|
||||
- H3 모델 비교(게임PC GPU, 워커와 같은 경로, int8_float16, 정답 16개 기준 문자 오류율): large-v3-turbo 12.0%, large-v3 12.7%, BuzzASR/korean 13.3%, batisay-ko-turbo 11.2%, ghost613 turbo-korean 33.1%. 모두 같은 곳을 틀림("기사야", "화면군이", "기구", "화면" 빠짐). 상대 20% 넘게 나은 모델이 없어서 모델은 바꾸지 않음. Qwen3-ASR은 VRAM(Strata와 함께 남는 약 2.5GB)에 들어가지 않아 서버 CPU에서 따로 봄.
|
||||
- H4 앞 대화 문맥: 12.0% → 7.5%(상대 38%), 빈 결과 그대로, 잡음 108개의 지어낸 말 그대로(16개), 최종 전사 평균 182ms(p95 253ms). 문맥만 주면 잡음에서 앞 대화 낱말을 씀(turbo 22개, large-v3 63개)이라 호출어 힌트와 같은 규칙으로 막음. 0.1.48에 넣음(voice_worker.spec.md "Conversation context").
|
||||
- 남은 것: 첫 낱말(호출어)과 짧은 말("기사야"→비서야, "기구"→디코)은 앞 대화가 없거나 너무 짧아 그대로. 호출 판정의 비슷한 말 허용이 호출어 쪽을 덮음.
|
||||
|
||||
@@ -32,7 +32,8 @@ class SttResult:
|
||||
|
||||
|
||||
class Transcriber(Protocol):
|
||||
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions) -> list[str]: ...
|
||||
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions,
|
||||
contexts: list[str] | None = None) -> list[str]: ...
|
||||
|
||||
|
||||
Callback = Callable[[SttResult], None]
|
||||
@@ -44,6 +45,7 @@ class _Job:
|
||||
audio: np.ndarray
|
||||
opts: DecodeOptions
|
||||
callback: Callback
|
||||
context: str = "" # finals only: the recent conversation, as the model's prompt
|
||||
|
||||
|
||||
class SttScheduler:
|
||||
@@ -74,8 +76,9 @@ class SttScheduler:
|
||||
pass
|
||||
self._executor.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
def submit_final(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback) -> None:
|
||||
self._finals.append(_Job(key, audio, opts, callback))
|
||||
def submit_final(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback,
|
||||
context: str = "") -> None:
|
||||
self._finals.append(_Job(key, audio, opts, callback, context))
|
||||
self._wake.set()
|
||||
|
||||
def submit_partial(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback) -> None:
|
||||
@@ -121,7 +124,7 @@ class SttScheduler:
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
texts = await loop.run_in_executor(self._executor, self.transcriber.transcribe, clips,
|
||||
batch[0].opts)
|
||||
batch[0].opts, [j.context for j in batch])
|
||||
error = None
|
||||
except Exception as e: # the model failing must not stop every speaker's STT
|
||||
texts, error = [""] * len(batch), f"{type(e).__name__}: {e}"
|
||||
|
||||
@@ -10,6 +10,7 @@ import itertools
|
||||
import json
|
||||
import secrets
|
||||
import time
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable
|
||||
|
||||
@@ -28,6 +29,7 @@ from .tts import TtsEngine, TtsRequest, TtsService
|
||||
CLOSE_BAD_HELLO = 4400
|
||||
CLOSE_BAD_TOKEN = 4401
|
||||
MAX_SPEAKERS = 32
|
||||
CONTEXT_SECONDS = 180 # how far back the STT's conversation context reaches
|
||||
|
||||
|
||||
def is_loopback(host: str) -> bool:
|
||||
@@ -274,6 +276,9 @@ class Session:
|
||||
self.chat_waiting: dict[str, asyncio.Future] = {}
|
||||
self._chat_ids = itertools.count(1)
|
||||
self.names: dict[str, str] = {}
|
||||
# What was said and answered lately in Jarvis's voice channel: (time, text), the STT's
|
||||
# conversation context (voice_worker.spec.md, "Conversation context"). Never logged.
|
||||
self.lines: deque[tuple[float, str]] = deque(maxlen=50)
|
||||
self.speakers: dict[str, _Speaker] = {}
|
||||
self.tts_requests: dict[str, TtsRequest] = {}
|
||||
# With recordings on: each spoken answer's text and mono samples, kept until it is made and the
|
||||
@@ -364,6 +369,15 @@ class Session:
|
||||
samples = st.converter.push(pcm)
|
||||
self._handle_events(speaker, st, st.segmenter.push(samples, time.monotonic()))
|
||||
|
||||
def _context(self) -> str:
|
||||
"""The recent conversation for a final's prompt: the channel's lines of the last
|
||||
CONTEXT_SECONDS, at most `context_chars` characters from the end."""
|
||||
n = self.settings.context_chars
|
||||
if not n or not self.voice_channel:
|
||||
return ""
|
||||
since = time.monotonic() - CONTEXT_SECONDS
|
||||
return " ".join(t for at, t in self.lines if at >= since)[-n:].strip()
|
||||
|
||||
def _opts(self, beam: int) -> DecodeOptions:
|
||||
return DecodeOptions(beam=beam, language=self.settings.language, hotwords=self.settings.hotwords)
|
||||
|
||||
@@ -388,7 +402,8 @@ class Session:
|
||||
else:
|
||||
sched.submit_final((self.sid, speaker), loudness_equalise(ev.audio, ev.voiced),
|
||||
self._opts(self.settings.final_beam),
|
||||
lambda res, ev=ev, said=said: self._send_final(speaker, ev, res, said))
|
||||
lambda res, ev=ev, said=said: self._send_final(speaker, ev, res, said),
|
||||
context=self._context())
|
||||
|
||||
def _partial_cb(self, speaker: str, st: _Speaker, utt: int):
|
||||
def cb(res: SttResult) -> None:
|
||||
@@ -402,6 +417,8 @@ class Session:
|
||||
store = self.worker.history
|
||||
if store is not None and channel and res.text.strip():
|
||||
store.add_voice(channel, speaker, name, res.text, ts)
|
||||
if channel and channel == self.voice_channel and res.text.strip():
|
||||
self.lines.append((time.monotonic(), res.text.strip()))
|
||||
if self.worker.recorder is not None:
|
||||
self.worker.recorder.said(ts, speaker, name, channel, res.text, ev.audio, SR)
|
||||
self.send({"type": "final", "speaker": speaker, "utt": ev.utt, "text": res.text,
|
||||
@@ -499,6 +516,8 @@ class Session:
|
||||
rec["text"] += text
|
||||
while len(self.tts_recorded) > 50: # never reported as played: not kept forever
|
||||
self.tts_recorded.pop(next(iter(self.tts_recorded)))
|
||||
if self.voice_channel and text.strip():
|
||||
self.lines.append((time.monotonic(), text.strip()))
|
||||
tts.feed(req, text, more)
|
||||
|
||||
def _on_tts_cancel(self, msg: dict) -> None:
|
||||
@@ -523,6 +542,7 @@ class Session:
|
||||
if self.voice_channel:
|
||||
debug_log(f"session {self.sid} left voice channel {self.voice_channel}", "voice_worker")
|
||||
self.voice_channel = None
|
||||
self.lines.clear()
|
||||
|
||||
def _on_voice_join(self, msg: dict) -> None:
|
||||
got = self._strings(msg, "channel")
|
||||
|
||||
@@ -13,6 +13,7 @@ class SttSettings:
|
||||
final_beam: int = 2
|
||||
language: str | None = "ko"
|
||||
hotwords: tuple[str, ...] = ()
|
||||
context_chars: int = 200
|
||||
|
||||
def update(self, changes: dict) -> None:
|
||||
"""Apply ``changes``; raises ValueError and changes nothing if any value is bad."""
|
||||
@@ -52,6 +53,8 @@ def _check(key: str, value):
|
||||
if value is not None and (not isinstance(value, str) or not 2 <= len(value) <= 8):
|
||||
raise ValueError("language must be a language code or null")
|
||||
return value
|
||||
if key == "context_chars":
|
||||
return _int_in(key, value, 0, 500)
|
||||
if key == "hotwords":
|
||||
if (not isinstance(value, (list, tuple)) or len(value) > 50
|
||||
or not all(isinstance(w, str) and 0 < len(w) <= 64 for w in value)):
|
||||
|
||||
@@ -73,6 +73,25 @@ def choose_with_hint(hint_text: str, hint_lp: float, plain_text: str, plain_lp:
|
||||
return plain_text
|
||||
|
||||
|
||||
def choose_with_context(heard: str, prompted: str, hotwords: tuple[str, ...]) -> str:
|
||||
"""Pick between the decode without the conversation context (``heard``, already through the hint
|
||||
choice) and the one that had it as Whisper's previous-text prompt (``prompted``).
|
||||
|
||||
The context fixes words of the topic the model misheard, but on noise or unsure audio it can write
|
||||
words of the prompt, so the decode without it decides, as for the hint: nothing heard stays
|
||||
nothing, and the prompted text is used only when it stays close to what was heard and holds every
|
||||
hotword exactly as often.
|
||||
"""
|
||||
if not heard or not prompted:
|
||||
return heard
|
||||
h, p = _letters(heard), _letters(prompted)
|
||||
if _edit_distance(p, h) > len(h) // 2:
|
||||
return heard
|
||||
if any(_mentions(prompted, w) != _mentions(heard, w) for w in hotwords):
|
||||
return heard
|
||||
return prompted
|
||||
|
||||
|
||||
def add_cuda_dll_dirs() -> None:
|
||||
"""On Windows ctranslate2 loads cuBLAS/cuDNN from the PATH; the pip wheels put them in nvidia/*/bin."""
|
||||
if not hasattr(os, "add_dll_directory"):
|
||||
@@ -104,8 +123,25 @@ class WhisperTranscriber:
|
||||
def warm_up(self) -> None:
|
||||
self.transcribe([np.zeros(SR, dtype=np.float32)], DecodeOptions(beam=1, language="en", hotwords=()))
|
||||
|
||||
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions) -> list[str]:
|
||||
"""Clips arrive loudness-equalised by the session."""
|
||||
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions,
|
||||
contexts: list[str] | None = None) -> list[str]:
|
||||
"""Clips arrive loudness-equalised by the session. ``contexts``: per clip, the recent
|
||||
conversation to prompt with ("" for none); see ``choose_with_context``."""
|
||||
if contexts and any(contexts):
|
||||
# Prompts of different lengths cannot share one generate call (the start token must sit
|
||||
# at the same position): one clip at a time. The encoder costs the same per clip anyway.
|
||||
return [self._with_context(c, opts, ctx) for c, ctx in zip(clips, contexts)]
|
||||
enc, langs = self._encode(clips, opts)
|
||||
return self._choose(enc, langs, opts, None)
|
||||
|
||||
def _with_context(self, clip: np.ndarray, opts: DecodeOptions, context: str) -> str:
|
||||
enc, langs = self._encode([clip], opts)
|
||||
heard = self._choose(enc, langs, opts, None)[0]
|
||||
if not context:
|
||||
return heard
|
||||
return choose_with_context(heard, self._choose(enc, langs, opts, [context])[0], opts.hotwords)
|
||||
|
||||
def _encode(self, clips: list[np.ndarray], opts: DecodeOptions):
|
||||
m = self.model
|
||||
n_frames = m.feature_extractor.nb_max_frames
|
||||
feats = []
|
||||
@@ -118,16 +154,22 @@ class WhisperTranscriber:
|
||||
langs = [opts.language] * len(clips)
|
||||
else:
|
||||
langs = [r[0][0][2:-2] for r in m.model.detect_language(enc)]
|
||||
plain = self._decode(enc, langs, opts.beam, None)
|
||||
return enc, langs
|
||||
|
||||
def _choose(self, enc, langs: list[str], opts: DecodeOptions, contexts: list[str] | None) -> list[str]:
|
||||
plain = self._decode(enc, langs, opts.beam, None, contexts)
|
||||
if not opts.hotwords:
|
||||
return [text if keep_transcript(text, lp) else "" for text, lp in plain]
|
||||
hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords))
|
||||
hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords), contexts)
|
||||
return [choose_with_hint(ht, hlp, pt, plp, opts.hotwords) for (ht, hlp), (pt, plp) in zip(hinted, plain)]
|
||||
|
||||
def _decode(self, enc, langs: list[str], beam: int, hotwords: str | None) -> list[tuple[str, float]]:
|
||||
def _decode(self, enc, langs: list[str], beam: int, hotwords: str | None,
|
||||
contexts: list[str] | None = None) -> list[tuple[str, float]]:
|
||||
m = self.model
|
||||
prompts = [m.get_prompt(self._tokenizer(lang), [], without_timestamps=True, hotwords=hotwords)
|
||||
for lang in langs]
|
||||
prompts = [m.get_prompt(self._tokenizer(lang),
|
||||
self._tokenizer(lang).encode(" " + ctx.strip()) if ctx else [],
|
||||
without_timestamps=True, hotwords=hotwords)
|
||||
for lang, ctx in zip(langs, contexts or [""] * len(langs))]
|
||||
results = m.model.generate(enc, prompts, beam_size=beam, max_length=224, suppress_blank=True,
|
||||
suppress_tokens=[-1], return_scores=True)
|
||||
# ``scores`` is the sequence log-probability divided by its length: the mean per token.
|
||||
|
||||
@@ -80,6 +80,7 @@ STT settings (all optional in `hello` and `config`):
|
||||
| `end_silence_ms` | 300 | Trailing VAD silence that ends an utterance. Range 100-2000. |
|
||||
| `final_beam` | 2 | Beam size for finals (partials always use 1). Range 1-5. |
|
||||
| `language` | `"ko"` | Whisper language code, or `null` to detect per utterance. |
|
||||
| `context_chars` | 200 | Finals are also decoded with the recent conversation as Whisper's previous-text prompt, at most this many characters (Conversation context). `0` turns it off. Range 0-500. |
|
||||
| `hotwords` | `[]` | Words (wake words, names) passed to Whisper as a hint. The hint alone made Whisper write the wake word on laughter, so with hotwords every clip is also decoded without them and that plain decode decides (see Hallucination gate). |
|
||||
|
||||
### Utterance rules
|
||||
@@ -152,6 +153,35 @@ text is empty:
|
||||
|
||||
`no_speech_prob` is not used: large-v3-turbo reports 0.0 for silence and noise alike.
|
||||
|
||||
### Conversation context
|
||||
|
||||
People talk about one thing for a while, and the words of that topic ("디코", "화면 공유", "계산기")
|
||||
were the ones misheard (live 2026-11-19): Whisper heard a casual, unclear "화면 공유" as "화면공이"
|
||||
and "계산기" as "계산지", and five models (large-v3-turbo, large-v3 and three Korean fine-tunes) made
|
||||
the same mistakes. Whisper was trained to continue from the text before a clip, so a final is also
|
||||
decoded with the recent conversation as its previous-text prompt:
|
||||
|
||||
- The context is what was said (the session's non-empty finals, everyone's) and what Jarvis said
|
||||
(the text of every `tts` request) in the voice channel Jarvis is in, over the last 180 s, joined
|
||||
by spaces, at most `context_chars` characters from the end. Joining another channel or leaving
|
||||
clears it. It lives in memory only, is never logged and goes nowhere but the model's prompt.
|
||||
- Partials have none (they stay one cheap decode).
|
||||
- The clip is decoded as before (with the hotword choice above) and again with the context (the
|
||||
same choice, each decode prompted). The prompted text is used only when the one without context
|
||||
was kept, it stays within half that text's letters of it, and it holds every hotword exactly as
|
||||
often; otherwise the text without context is used. On noise or unsure audio the prompt makes
|
||||
Whisper write words of the conversation; this rule keeps that out, as for the hint.
|
||||
- Prompts of different lengths cannot share one decoder call, so finals with a context are decoded
|
||||
one clip at a time (the encoder costs the same per clip).
|
||||
|
||||
Measured (2026-11-19, the game PC's RTX 5070 Ti with Strata loaded, large-v3-turbo int8_float16,
|
||||
beam 2, the 100 recordings of the user's speech from 2026-11-16 to 19, 16 of them with the user's own
|
||||
text): character error rate 12.0% without and 7.5% with the context (wrong sentences 10 to 8), no more
|
||||
empty results (17 of 100 both ways), final decode 134 ms to 182 ms on average (p95 170 to 253 ms).
|
||||
On the 108 noise clips decoded straight (no VAD), as many came out with text with the context as
|
||||
without it (16 each); without the rule above it was 22 with turbo, and with large-v3 63, which wrote
|
||||
whole sentences of the prompt. Evidence and the model comparison: `docs/v2/stt-accuracy-plan.md`.
|
||||
|
||||
Measured (2026-11-11, RTX 5050) with 54 real noise recordings from ESC-50 (typing, mouse, breathing,
|
||||
coughing, laughing, crying, vacuum, ...), each at its own level and at -25 dBFS: of the 108 noise-only
|
||||
clips streamed through the worker with the wake word hint on, 103 never reached the model (the VAD
|
||||
|
||||
@@ -303,7 +303,7 @@ class SlowTranscriber(LevelTranscriber):
|
||||
super().__init__()
|
||||
self.delay = delay
|
||||
|
||||
def transcribe(self, clips, opts):
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
import time as _t
|
||||
_t.sleep(self.delay)
|
||||
return super().transcribe(clips, opts)
|
||||
|
||||
@@ -96,3 +96,39 @@ def test_wake_word_matching_ignores_case_spacing_and_punctuation():
|
||||
def test_spacing_and_punctuation_differences_do_not_change_the_choice():
|
||||
assert choose_with_hint("자비스 오늘 서울 날씨 어때", -0.1, "자비스, 오늘 서울 날씨 어때?", -0.1, HINT) \
|
||||
== "자비스 오늘 서울 날씨 어때"
|
||||
|
||||
|
||||
# The recent conversation as Whisper's previous-text prompt (voice_worker.spec.md, "Conversation
|
||||
# context"). Live 2026-11-19 the prompt fixed words of the topic ("화면공이" -> "화면 공유", "계산지" ->
|
||||
# "계산기"); on noise it wrote words of the prompt. The decode without it decides, as with the hint.
|
||||
from voice_worker.stt_whisper import choose_with_context # noqa: E402
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_context_may_fix_words_of_what_was_said():
|
||||
assert choose_with_context("내 화면공이 안 보여?", "내 화면 공유 안 보여?", HINT) == "내 화면 공유 안 보여?"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_nothing_heard_without_the_context_stays_nothing():
|
||||
# Noise: the prompted decode wrote words of the conversation.
|
||||
assert choose_with_context("", "화면에 디스코드", HINT) == ""
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_prompted_decode_far_from_what_was_heard_is_not_used():
|
||||
heard = "가상화면을 못 봐"
|
||||
copied = "네, 현재 화면에 디스코드 채팅 창과 여러 프로그램이 정상적으로 보입니다."
|
||||
assert choose_with_context(heard, copied, HINT) == heard
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_context_never_changes_how_many_wake_words_were_heard():
|
||||
assert choose_with_context("라비스 지금 영상 재생되고 있어?", "자비스 지금 영상 재생되고 있어?", HINT) \
|
||||
== "라비스 지금 영상 재생되고 있어?"
|
||||
assert choose_with_context("자비스 날씨 알려 줘", "날씨 알려 줘", HINT) == "자비스 날씨 알려 줘"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_an_empty_prompted_decode_keeps_what_was_heard():
|
||||
assert choose_with_context("네.", "", HINT) == "네."
|
||||
|
||||
@@ -25,7 +25,7 @@ class FakeTranscriber:
|
||||
self.gate.set()
|
||||
self.fail_on_beam = fail_on_beam
|
||||
|
||||
def transcribe(self, clips, opts):
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
self.entered.set()
|
||||
self.gate.wait(5)
|
||||
self.calls.append((opts, [len(c) for c in clips]))
|
||||
@@ -218,12 +218,45 @@ class StepTranscriber:
|
||||
def step(self):
|
||||
self._steps.release()
|
||||
|
||||
def transcribe(self, clips, opts):
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
self.calls.append([len(c) for c in clips])
|
||||
self._steps.acquire(timeout=5)
|
||||
return ["" for _ in clips]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_each_final_reaches_the_model_with_its_own_context():
|
||||
class Ctx:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
self.calls.append(list(contexts or []))
|
||||
return [f"n{len(c)}" for c in clips]
|
||||
|
||||
eng = Ctx()
|
||||
|
||||
async def body():
|
||||
s = SttScheduler(eng)
|
||||
s.start()
|
||||
got = []
|
||||
done = asyncio.Event()
|
||||
|
||||
def cb(r):
|
||||
got.append(r.text)
|
||||
if len(got) == 3:
|
||||
done.set()
|
||||
s.submit_final("a", clip(100), FINAL, cb, context="화면 공유")
|
||||
s.submit_final("b", clip(200), FINAL, cb)
|
||||
s.submit_partial("c", clip(300), PARTIAL, cb)
|
||||
await asyncio.wait_for(done.wait(), 5)
|
||||
await s.stop()
|
||||
|
||||
asyncio.run(body())
|
||||
assert eng.calls[0] == ["화면 공유", ""]
|
||||
assert eng.calls[1] == [""], "partials go without context"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_partial_batches_are_kept_small_so_finals_can_cut_in():
|
||||
t = StepTranscriber()
|
||||
|
||||
@@ -32,7 +32,7 @@ class LevelTranscriber:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def transcribe(self, clips, opts):
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
self.calls.append((opts, len(clips)))
|
||||
return [f"level {np.argmax(np.abs(np.fft.rfft(c))) * 16000 / len(c) / 1000:.1f}" for c in clips]
|
||||
|
||||
@@ -336,7 +336,7 @@ def test_48k_stereo_input_reaches_the_model_at_16k():
|
||||
lengths = []
|
||||
orig = t.transcribe
|
||||
|
||||
def spy(clips, opts):
|
||||
def spy(clips, opts, contexts=None):
|
||||
lengths.extend(len(c) for c in clips)
|
||||
return orig(clips, opts)
|
||||
|
||||
@@ -496,3 +496,47 @@ def test_old_recordings_are_also_removed_while_the_worker_keeps_running(tmp_path
|
||||
r.said(later.timestamp(), "1", "A", "vc", "나중", np.zeros(1600))
|
||||
r.close()
|
||||
assert sorted(p.name for p in tmp_path.iterdir()) == [later.date().isoformat()]
|
||||
|
||||
|
||||
class ContextTranscriber(LevelTranscriber):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.contexts = []
|
||||
|
||||
def transcribe(self, clips, opts, contexts=None):
|
||||
if opts.beam > 1: # finals
|
||||
self.contexts += list(contexts or [""] * len(clips))
|
||||
return super().transcribe(clips, opts)
|
||||
|
||||
|
||||
async def _said(ws, speaker, level):
|
||||
await stream(ws, speaker, level, 600)
|
||||
await ws.send(json.dumps({"type": "speaking_end", "speaker": speaker}))
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final" and m["speaker"] == speaker)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_each_final_is_transcribed_with_the_recent_conversation_of_the_channel():
|
||||
"""voice_worker.spec.md, "Conversation context": what was said and answered just before, in the
|
||||
channel Jarvis is in, is the model's prompt for the next final."""
|
||||
t = ContextTranscriber()
|
||||
|
||||
async def body(url):
|
||||
async with connect(url) as ws:
|
||||
await hello(ws, stt={"partials": False})
|
||||
await ws.send(json.dumps({"type": "voice_join", "channel": "vc1"}))
|
||||
await _said(ws, "100", 0.9)
|
||||
await ws.send(json.dumps({"type": "tts", "id": "r1", "text": "디스코드 화면 공유가 보입니다.",
|
||||
"more": False}))
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
|
||||
await _said(ws, "200", 0.6)
|
||||
await ws.send(json.dumps({"type": "voice_join", "channel": "vc2"})) # another channel
|
||||
await _said(ws, "100", 0.9)
|
||||
await ws.send(json.dumps({"type": "config", "context_chars": 0}))
|
||||
await _said(ws, "100", 0.6)
|
||||
|
||||
asyncio.run(with_worker(body, transcriber=t, tts=FakeTts()))
|
||||
assert t.contexts[0] == ""
|
||||
assert t.contexts[1] == "level 0.9 디스코드 화면 공유가 보입니다."
|
||||
assert t.contexts[2] == "", "a new channel starts without the old channel's conversation"
|
||||
assert t.contexts[3] == "", "context_chars 0 turns it off"
|
||||
|
||||
Reference in New Issue
Block a user