Files
javis_bot/scripts/m0/m0lib/stt.py
javis-bot 3794bb7149 feat(m0): measure the LLM next to the STT, the STT on the CPU and a real game
gamemode now also starts the 2 GB expert placement with a 16K context and
drops the --kv q4_0 and --mtp off rows the Strata 0.1.59 qwen36 engine
ignores; the normal placement answers one request at a time.

gamemode_stt loads the STT worker (stt.py mode=load) before the LLM picks its
placement and keeps it transcribing ten speakers at once while the LLM is
measured, so the VRAM left is the real game share. The game placements run
with the STT on the GPU and on the CPU. gamemode_live repeats the game rows
while the user runs a real game; it only runs when asked for
(run_m0_game_live.bat) and is skipped when nobody is at the PC.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 19:12:53 +09:00

229 lines
10 KiB
Python

"""Stage: faster-whisper large-v3-turbo speed on the GPU.
Runs inside its own venv (faster-whisper, nvidia-cublas-cu12, nvidia-cudnn-cu12).
Audio: the synthetic Korean samples in assets/ko_samples (MeloTTS voice).
With --opt mode=load it is instead the STT load the gamemode stages put next to
the LLM: load the model on device=cuda|cpu, write {"status": "loaded"} to --out,
wait for the file go=PATH, then transcribe ten speakers at once (and one final
clip) on a loop until the file stop=PATH appears, and write the timings.
"""
from __future__ import annotations
import glob
import json
import os
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from m0lib.common import (GpuSampler, StageResult, cer, gpu_memory_mib, say, stage_args, stats, # noqa: E402
write_json)
SR = 16000
ASSETS = Path(__file__).resolve().parent.parent / "assets" / "ko_samples"
def add_cuda_dll_dirs() -> list[str]:
"""ctranslate2 loads cuBLAS/cuDNN from the PATH on Windows; the pip wheels put them in nvidia/*/bin."""
found = []
for sp in sys.path:
for d in glob.glob(os.path.join(sp, "nvidia", "*", "bin")):
if os.path.isdir(d) and d not in found:
found.append(d)
for d in found:
os.environ["PATH"] = d + os.pathsep + os.environ.get("PATH", "")
if hasattr(os, "add_dll_directory"):
os.add_dll_directory(d)
return found
def read_wav(path: Path):
"""The kit's samples are 16 kHz mono PCM16. Read them with the standard library:
faster-whisper's PyAV path broke on Windows with a newer PyAV
(open() got an unexpected keyword argument 'metadata_errors')."""
import wave
import numpy as np
with wave.open(str(path), "rb") as w:
assert w.getframerate() == SR and w.getnchannels() == 1 and w.getsampwidth() == 2, path
return np.frombuffer(w.readframes(w.getnframes()), dtype="<i2").astype(np.float32) / 32768.0
def transcribe(model, audio, beam: int, hotwords: str | None = None) -> tuple[str, float]:
t0 = time.perf_counter()
segs, _ = model.transcribe(audio, language="ko", beam_size=beam, vad_filter=False,
condition_on_previous_text=False, without_timestamps=True, hotwords=hotwords)
text = "".join(s.text for s in segs).strip()
return text, time.perf_counter() - t0
def batch_transcribe(model, audios: list, beam: int = 1) -> tuple[list[str], float]:
"""One encoder + decoder pass for several speakers' clips (the v2 design: one shared model)."""
import numpy as np
from faster_whisper.tokenizer import Tokenizer
t0 = time.perf_counter()
feats = []
for a in audios:
a30 = np.pad(a[: SR * 30], (0, max(0, SR * 30 - len(a))))
feats.append(model.feature_extractor(a30, padding=0)[:, :3000])
enc = model.encode(np.stack(feats).astype(np.float32))
tok = Tokenizer(model.hf_tokenizer, model.model.is_multilingual, task="transcribe", language="ko")
prompt = model.get_prompt(tok, previous_tokens=[], without_timestamps=True)
res = model.model.generate(enc, [prompt] * len(audios), beam_size=beam, max_length=224, suppress_blank=True,
suppress_tokens=[-1])
texts = [tok.decode(x.sequences_ids[0]).strip() for x in res]
return texts, time.perf_counter() - t0
def load_loop(batch, single, go: Path, stop: Path, wait_s: float = 3600, max_s: float = 600) -> dict:
"""Wait for ``go``, then run ``batch()`` (ten speakers) and ``single()`` (one final clip)
in turn until ``stop`` appears. An error (out of memory) ends the loop and is kept."""
t0 = time.monotonic()
while not go.exists():
if stop.exists() or time.monotonic() - t0 > wait_s:
return {"status": "not_started", "batch10_s": stats([]), "single_s": stats([]), "errors": []}
time.sleep(0.2)
b, one, errors = [], [], []
t1 = time.monotonic()
while not stop.exists() and time.monotonic() - t1 < max_s:
try:
for fn, into in ((batch, b), (single, one)):
s0 = time.perf_counter()
fn()
into.append(time.perf_counter() - s0)
except Exception as e:
errors.append(f"{type(e).__name__}: {str(e)[:300]}")
break
return {"status": "done", "batch10_s": stats(b), "single_s": stats(one), "errors": errors,
"seconds": round(time.monotonic() - t1, 1)}
def load_worker(out: Path, work: Path, opts: dict) -> int:
device = opts.get("device", "cuda")
ctype = opts.get("compute", "int8_float16" if device == "cuda" else "int8")
threads = int(opts.get("threads", "8"))
res: dict = {"device": device, "compute_type": ctype, "cpu_threads": threads if device == "cpu" else None}
try:
if device == "cuda":
add_cuda_dll_dirs()
from faster_whisper import WhisperModel
refs = json.loads((ASSETS / "refs.json").read_text(encoding="utf-8"))["samples"]
clips = [read_wav(ASSETS / refs[i % len(refs)]["file"]) for i in range(10)]
t0 = time.perf_counter()
model = WhisperModel(opts.get("model", "large-v3-turbo"), device=device, compute_type=ctype,
cpu_threads=threads if device == "cpu" else 0,
download_root=str(work / "models" / "stt"))
transcribe(model, clips[0], 1) # warm-up
res["load_s"] = round(time.perf_counter() - t0, 2)
except Exception as e:
write_json(out, dict(res, status="failed", errors=[f"load: {type(e).__name__}: {str(e)[:300]}"]))
return 0
write_json(out, dict(res, status="loaded"))
res.update(load_loop(lambda: batch_transcribe(model, clips), lambda: transcribe(model, clips[0], 1),
Path(opts["go"]), Path(opts["stop"])))
write_json(out, res)
return 0
def main() -> int:
out, work, opts = stage_args()
if opts.get("mode") == "load":
return load_worker(out, work, opts)
r = StageResult("stt", out)
r["cuda_dll_dirs"] = len(add_cuda_dll_dirs())
os.environ.setdefault("HF_HOME", str(work / "models" / "hf"))
try:
from faster_whisper import WhisperModel
import ctranslate2
r["versions"] = {"ctranslate2": ctranslate2.__version__,
"faster_whisper": __import__("faster_whisper").__version__,
"cuda_devices": ctranslate2.get_cuda_device_count()}
except Exception as e:
r.error("import", e)
r.save("failed")
return 0
refs = json.loads((ASSETS / "refs.json").read_text(encoding="utf-8"))["samples"]
audio = {s["file"]: read_wav(ASSETS / s["file"]) for s in refs}
name = opts.get("model", "large-v3-turbo")
r["model"] = name
r["audio_source"] = "synthetic MeloTTS Korean (clean speech; real voices will be harder)"
for ctype in opts.get("compute_types", "int8_float16,float16").split(","):
res: dict = {}
r[f"run_{ctype}"] = res
say(f" - {name} / {ctype}")
try:
before = gpu_memory_mib()
t0 = time.perf_counter()
model = WhisperModel(name, device="cuda", compute_type=ctype, download_root=str(work / "models" / "stt"))
res["load_s"] = round(time.perf_counter() - t0, 2)
transcribe(model, audio[refs[0]["file"]], 1) # warm-up (CUDA kernels)
after = gpu_memory_mib()
if before and after:
res["vram_model_mib"] = after[0] - before[0]
say(f" 로딩 {res['load_s']}s, VRAM 약 {res.get('vram_model_mib')} MiB")
except Exception as e:
r.error(f"load {ctype}", e)
continue
# Final transcription, fast (beam 1) and accurate (beam 3).
for beam in (1, 3):
rows = []
for s in refs:
text, sec = transcribe(model, audio[s["file"]], beam)
rows.append({"file": s["file"], "audio_s": s["seconds"], "seconds": round(sec, 3),
"cer": cer(s["text"], text), "text": text})
res[f"final_beam{beam}"] = {"latency": stats(x["seconds"] for x in rows),
"cer": stats([x["cer"] for x in rows], 4), "rows": rows}
say(f" 최종 전사 beam {beam}: 중앙값 {res[f'final_beam{beam}']['latency']['median']}s, "
f"CER {res[f'final_beam{beam}']['cer']['median']}")
# Wake-word hint.
rows = []
for s in refs:
if "자비스" in s["text"]:
plain, _ = transcribe(model, audio[s["file"]], 1)
hinted, _ = transcribe(model, audio[s["file"]], 1, hotwords="자비스")
rows.append({"file": s["file"], "plain": plain, "hotword": hinted,
"plain_has": "자비스" in plain, "hotword_has": "자비스" in hinted})
res["wake_word"] = rows
# Streaming: re-transcribe the growing buffer every 0.5 s (beam 1).
long = max(refs, key=lambda s: s["seconds"])
a = audio[long["file"]]
lat = []
for end in range(SR // 2, len(a) + 1, SR // 2):
_, sec = transcribe(model, a[:end], 1)
lat.append(round(sec, 3))
res["partial_every_0_5s"] = {"audio_s": long["seconds"], "latency": stats(lat), "all": lat}
# The last piece only (what v2 decodes after the end of speech).
_, sec = transcribe(model, a[-SR * 2:], 1)
res["last_2s_piece_s"] = round(sec, 3)
# One shared model for several speakers at once.
batch = {}
with GpuSampler(interval=0.2) as g:
for n in (1, 4, 10):
clips = [audio[refs[i % len(refs)]["file"]] for i in range(n)]
try:
batch_transcribe(model, clips) # warm-up for this batch size
texts, sec = batch_transcribe(model, clips)
errs = [cer(refs[i % len(refs)]["text"], t) for i, t in enumerate(texts)]
batch[str(n)] = {"seconds": round(sec, 3), "cer": stats(errs, 4), "sample_text": texts[0]}
say(f" {n}명 동시(배치): {batch[str(n)]['seconds']}s")
except Exception as e:
r.error(f"batch {n}", e)
batch.update(g.summary())
res["batch_speakers"] = batch
del model
r.save()
r.save("ok" if not r["errors"] else "partial")
return 0
if __name__ == "__main__":
sys.exit(main())