Files
javis_bot/scripts/m0/m0lib/stt.py
javis-bot 653d6ecc03 feat(m0): add gaming-PC measurement kit for v2 defaults
One double-click kit (scripts/m0/run_m0.bat) that measures the gaming PC
and packs the results into a single zip:

- PC info (GPU, driver, RAM, disks, monitors)
- Windows checks with the user: bot browser window hidden from Alt+Tab
  but listed in Discord's share picker and still drawing while covered;
  Discord pop-out stream window identification and occlusion; installed
  games, anti-cheat services and the borderless game detector
- MeloTTS, CosyVoice2 and Fun-CosyVoice3 Korean synthesis speed
- faster-whisper large-v3-turbo: final/partial latency, CER, wake-word
  hint, batched decoding for several speakers
- Strata LLM (fast and recommended presets): TTFT by context length,
  decode speed, tool calls, picture reading per quality tier, Korean
  answers, parallel requests, VRAM give-back and restart/switch timings

uv and Python, per-stage venvs, models and Strata all live inside the
work folder. Results are scrubbed of user/computer names and home paths.
Verified on Linux GPU (LLM client against Ollama, STT, MeloTTS,
CosyVoice2/3) and on the Windows 11 CI VM (window checks, game scan,
MeloTTS/CosyVoice2/STT installs, Strata install and plan).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-09 15:38:41 +09:00

171 lines
7.5 KiB
Python

"""Stage: faster-whisper large-v3-turbo speed on the GPU.
Runs inside its own venv (faster-whisper, nvidia-cublas-cu12, nvidia-cudnn-cu12).
Audio: the synthetic Korean samples in assets/ko_samples (MeloTTS voice).
"""
from __future__ import annotations
import glob
import json
import os
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from m0lib.common import GpuSampler, StageResult, cer, gpu_memory_mib, say, stage_args, stats # noqa: E402
SR = 16000
ASSETS = Path(__file__).resolve().parent.parent / "assets" / "ko_samples"
def add_cuda_dll_dirs() -> list[str]:
"""ctranslate2 loads cuBLAS/cuDNN from the PATH on Windows; the pip wheels put them in nvidia/*/bin."""
found = []
for sp in sys.path:
for d in glob.glob(os.path.join(sp, "nvidia", "*", "bin")):
if os.path.isdir(d) and d not in found:
found.append(d)
for d in found:
os.environ["PATH"] = d + os.pathsep + os.environ.get("PATH", "")
if hasattr(os, "add_dll_directory"):
os.add_dll_directory(d)
return found
def read_wav(path: Path):
"""The kit's samples are 16 kHz mono PCM16. Read them with the standard library:
faster-whisper's PyAV path broke on Windows with a newer PyAV
(open() got an unexpected keyword argument 'metadata_errors')."""
import wave
import numpy as np
with wave.open(str(path), "rb") as w:
assert w.getframerate() == SR and w.getnchannels() == 1 and w.getsampwidth() == 2, path
return np.frombuffer(w.readframes(w.getnframes()), dtype="<i2").astype(np.float32) / 32768.0
def transcribe(model, audio, beam: int, hotwords: str | None = None) -> tuple[str, float]:
t0 = time.perf_counter()
segs, _ = model.transcribe(audio, language="ko", beam_size=beam, vad_filter=False,
condition_on_previous_text=False, without_timestamps=True, hotwords=hotwords)
text = "".join(s.text for s in segs).strip()
return text, time.perf_counter() - t0
def batch_transcribe(model, audios: list, beam: int = 1) -> tuple[list[str], float]:
"""One encoder + decoder pass for several speakers' clips (the v2 design: one shared model)."""
import numpy as np
from faster_whisper.tokenizer import Tokenizer
t0 = time.perf_counter()
feats = []
for a in audios:
a30 = np.pad(a[: SR * 30], (0, max(0, SR * 30 - len(a))))
feats.append(model.feature_extractor(a30, padding=0)[:, :3000])
enc = model.encode(np.stack(feats).astype(np.float32))
tok = Tokenizer(model.hf_tokenizer, model.model.is_multilingual, task="transcribe", language="ko")
prompt = model.get_prompt(tok, previous_tokens=[], without_timestamps=True)
res = model.model.generate(enc, [prompt] * len(audios), beam_size=beam, max_length=224, suppress_blank=True,
suppress_tokens=[-1])
texts = [tok.decode(x.sequences_ids[0]).strip() for x in res]
return texts, time.perf_counter() - t0
def main() -> int:
out, work, opts = stage_args()
r = StageResult("stt", out)
r["cuda_dll_dirs"] = len(add_cuda_dll_dirs())
os.environ.setdefault("HF_HOME", str(work / "models" / "hf"))
try:
from faster_whisper import WhisperModel
import ctranslate2
r["versions"] = {"ctranslate2": ctranslate2.__version__,
"faster_whisper": __import__("faster_whisper").__version__,
"cuda_devices": ctranslate2.get_cuda_device_count()}
except Exception as e:
r.error("import", e)
r.save("failed")
return 0
refs = json.loads((ASSETS / "refs.json").read_text(encoding="utf-8"))["samples"]
audio = {s["file"]: read_wav(ASSETS / s["file"]) for s in refs}
name = opts.get("model", "large-v3-turbo")
r["model"] = name
r["audio_source"] = "synthetic MeloTTS Korean (clean speech; real voices will be harder)"
for ctype in opts.get("compute_types", "int8_float16,float16").split(","):
res: dict = {}
r[f"run_{ctype}"] = res
say(f" - {name} / {ctype}")
try:
before = gpu_memory_mib()
t0 = time.perf_counter()
model = WhisperModel(name, device="cuda", compute_type=ctype, download_root=str(work / "models" / "stt"))
res["load_s"] = round(time.perf_counter() - t0, 2)
transcribe(model, audio[refs[0]["file"]], 1) # warm-up (CUDA kernels)
after = gpu_memory_mib()
if before and after:
res["vram_model_mib"] = after[0] - before[0]
say(f" 로딩 {res['load_s']}s, VRAM 약 {res.get('vram_model_mib')} MiB")
except Exception as e:
r.error(f"load {ctype}", e)
continue
# Final transcription, fast (beam 1) and accurate (beam 3).
for beam in (1, 3):
rows = []
for s in refs:
text, sec = transcribe(model, audio[s["file"]], beam)
rows.append({"file": s["file"], "audio_s": s["seconds"], "seconds": round(sec, 3),
"cer": cer(s["text"], text), "text": text})
res[f"final_beam{beam}"] = {"latency": stats(x["seconds"] for x in rows),
"cer": stats([x["cer"] for x in rows], 4), "rows": rows}
say(f" 최종 전사 beam {beam}: 중앙값 {res[f'final_beam{beam}']['latency']['median']}s, "
f"CER {res[f'final_beam{beam}']['cer']['median']}")
# Wake-word hint.
rows = []
for s in refs:
if "자비스" in s["text"]:
plain, _ = transcribe(model, audio[s["file"]], 1)
hinted, _ = transcribe(model, audio[s["file"]], 1, hotwords="자비스")
rows.append({"file": s["file"], "plain": plain, "hotword": hinted,
"plain_has": "자비스" in plain, "hotword_has": "자비스" in hinted})
res["wake_word"] = rows
# Streaming: re-transcribe the growing buffer every 0.5 s (beam 1).
long = max(refs, key=lambda s: s["seconds"])
a = audio[long["file"]]
lat = []
for end in range(SR // 2, len(a) + 1, SR // 2):
_, sec = transcribe(model, a[:end], 1)
lat.append(round(sec, 3))
res["partial_every_0_5s"] = {"audio_s": long["seconds"], "latency": stats(lat), "all": lat}
# The last piece only (what v2 decodes after the end of speech).
_, sec = transcribe(model, a[-SR * 2:], 1)
res["last_2s_piece_s"] = round(sec, 3)
# One shared model for several speakers at once.
batch = {}
with GpuSampler(interval=0.2) as g:
for n in (1, 4, 10):
clips = [audio[refs[i % len(refs)]["file"]] for i in range(n)]
try:
batch_transcribe(model, clips) # warm-up for this batch size
texts, sec = batch_transcribe(model, clips)
errs = [cer(refs[i % len(refs)]["text"], t) for i, t in enumerate(texts)]
batch[str(n)] = {"seconds": round(sec, 3), "cer": stats(errs, 4), "sample_text": texts[0]}
say(f" {n}명 동시(배치): {batch[str(n)]['seconds']}s")
except Exception as e:
r.error(f"batch {n}", e)
batch.update(g.summary())
res["batch_speakers"] = batch
del model
r.save()
r.save("ok" if not r["errors"] else "partial")
return 0
if __name__ == "__main__":
sys.exit(main())