Files
javis_bot/scripts/m0/m0lib/stt.py
javis-bot 2a85b6afa5 feat(m0): measure the normal placement with room for the STT and paced STT partials
The gamemode_stt run showed Strata's automatic placement filling the card
next to the STT (15.9 of 16.3 GB, first token 1.6 -> 7.5 s) and the STT's
back-to-back partial passes cutting LLM output to a third. The new
gamemode_tune stage measures both fixes the user chose:

- stt_room: one start without experts (STT loaded) gives the LLM's base
  VRAM; --expert-vram gets what is left after the STT's ten-speaker
  growth (from this PC's stt stage) and a 768 MiB margin.
- stt_pace: a ten-speaker partial pass every 2 s, or none, with finals
  every 0.5 s and finals first; records final wait and STT GPU share.

run_m0_gamemode_tune.bat runs it unattended.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 21:35:23 +09:00

278 lines
12 KiB
Python

"""Stage: faster-whisper large-v3-turbo speed on the GPU.
Runs inside its own venv (faster-whisper, nvidia-cublas-cu12, nvidia-cudnn-cu12).
Audio: the synthetic Korean samples in assets/ko_samples (MeloTTS voice).
With --opt mode=load it is instead the STT load the gamemode stages put next to
the LLM: load the model on device=cuda|cpu, write {"status": "loaded"} to --out,
wait for the file go=PATH, then transcribe ten speakers at once (and one final
clip) on a loop until the file stop=PATH appears, and write the timings.
partial_every=SECONDS|none and final_every=SECONDS pace that loop the way the app
would while the LLM answers: a ten-speaker partial pass every N seconds (none: no
partials) and a final clip every M seconds in between.
"""
from __future__ import annotations
import glob
import json
import os
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from m0lib.common import (GpuSampler, StageResult, cer, gpu_memory_mib, say, stage_args, stats, # noqa: E402
write_json)
SR = 16000
ASSETS = Path(__file__).resolve().parent.parent / "assets" / "ko_samples"
def add_cuda_dll_dirs() -> list[str]:
"""ctranslate2 loads cuBLAS/cuDNN from the PATH on Windows; the pip wheels put them in nvidia/*/bin."""
found = []
for sp in sys.path:
for d in glob.glob(os.path.join(sp, "nvidia", "*", "bin")):
if os.path.isdir(d) and d not in found:
found.append(d)
for d in found:
os.environ["PATH"] = d + os.pathsep + os.environ.get("PATH", "")
if hasattr(os, "add_dll_directory"):
os.add_dll_directory(d)
return found
def read_wav(path: Path):
"""The kit's samples are 16 kHz mono PCM16. Read them with the standard library:
faster-whisper's PyAV path broke on Windows with a newer PyAV
(open() got an unexpected keyword argument 'metadata_errors')."""
import wave
import numpy as np
with wave.open(str(path), "rb") as w:
assert w.getframerate() == SR and w.getnchannels() == 1 and w.getsampwidth() == 2, path
return np.frombuffer(w.readframes(w.getnframes()), dtype="<i2").astype(np.float32) / 32768.0
def transcribe(model, audio, beam: int, hotwords: str | None = None) -> tuple[str, float]:
t0 = time.perf_counter()
segs, _ = model.transcribe(audio, language="ko", beam_size=beam, vad_filter=False,
condition_on_previous_text=False, without_timestamps=True, hotwords=hotwords)
text = "".join(s.text for s in segs).strip()
return text, time.perf_counter() - t0
def batch_transcribe(model, audios: list, beam: int = 1) -> tuple[list[str], float]:
"""One encoder + decoder pass for several speakers' clips (the v2 design: one shared model)."""
import numpy as np
from faster_whisper.tokenizer import Tokenizer
t0 = time.perf_counter()
feats = []
for a in audios:
a30 = np.pad(a[: SR * 30], (0, max(0, SR * 30 - len(a))))
feats.append(model.feature_extractor(a30, padding=0)[:, :3000])
enc = model.encode(np.stack(feats).astype(np.float32))
tok = Tokenizer(model.hf_tokenizer, model.model.is_multilingual, task="transcribe", language="ko")
prompt = model.get_prompt(tok, previous_tokens=[], without_timestamps=True)
res = model.model.generate(enc, [prompt] * len(audios), beam_size=beam, max_length=224, suppress_blank=True,
suppress_tokens=[-1])
texts = [tok.decode(x.sequences_ids[0]).strip() for x in res]
return texts, time.perf_counter() - t0
def load_loop(batch, single, go: Path, stop: Path, pace: dict | None = None, wait_s: float = 3600,
max_s: float = 600) -> dict:
"""Wait for ``go``, then run ``batch()`` (a ten-speaker partial pass) and ``single()`` (one final
clip) until ``stop`` appears. Without ``pace`` they run back to back (every speaker talking
without a break). With ``pace`` a partial pass starts every ``partial_every`` s (None: none) and a
final every ``final_every`` s. A final that is due goes first; one that falls due during a
partial pass waits for it (final_wait_s).
An error (out of memory) ends the loop and is kept."""
t0 = time.monotonic()
while not go.exists():
if stop.exists() or time.monotonic() - t0 > wait_s:
return {"status": "not_started", "batch10_s": stats([]), "single_s": stats([]), "errors": [],
"pace": pace}
time.sleep(0.2)
# perf_counter: time.monotonic ticks every ~16 ms on Windows, too coarse for a 0.1 s clip.
b, one, waits, errors = [], [], [], []
t1 = time.perf_counter()
every_p = (pace or {}).get("partial_every")
every_f = (pace or {}).get("final_every")
next_p, next_f = t1, t1
def timed(fn, into):
s0 = time.perf_counter()
fn()
into.append(time.perf_counter() - s0)
while not stop.exists() and time.perf_counter() - t1 < max_s:
try:
if pace is None:
timed(batch, b)
timed(single, one)
continue
now = time.perf_counter()
# A final goes first: the answer waits for it, not for the subtitles.
if every_f is not None and now >= next_f:
waits.append(now - next_f)
next_f = max(next_f + every_f, now)
timed(single, one)
continue
if every_p is not None and now >= next_p:
next_p = now + every_p
timed(batch, b)
continue
dues = [x for x, on in ((next_p, every_p is not None), (next_f, every_f is not None)) if on]
time.sleep(min(max(min(dues, default=now + 0.1) - now, 0.0), 0.1))
except Exception as e:
errors.append(f"{type(e).__name__}: {str(e)[:300]}")
break
seconds = time.perf_counter() - t1
res = {"status": "done", "batch10_s": stats(b), "single_s": stats(one), "errors": errors,
"seconds": round(seconds, 1), "pace": pace,
"busy_share": round(min((sum(b) + sum(one)) / seconds, 1.0), 3) if seconds > 0 else None}
if pace is not None:
res["final_wait_s"] = stats(waits)
return res
def parse_pace(opts: dict) -> dict | None:
"""partial_every=SECONDS|none, final_every=SECONDS; neither: back to back (None)."""
if "partial_every" not in opts and "final_every" not in opts:
return None
def num(key):
v = opts.get(key, "none")
return None if v == "none" else float(v)
return {"partial_every": num("partial_every"), "final_every": num("final_every")}
def load_worker(out: Path, work: Path, opts: dict) -> int:
device = opts.get("device", "cuda")
ctype = opts.get("compute", "int8_float16" if device == "cuda" else "int8")
threads = int(opts.get("threads", "8"))
res: dict = {"device": device, "compute_type": ctype, "cpu_threads": threads if device == "cpu" else None}
try:
if device == "cuda":
add_cuda_dll_dirs()
from faster_whisper import WhisperModel
refs = json.loads((ASSETS / "refs.json").read_text(encoding="utf-8"))["samples"]
clips = [read_wav(ASSETS / refs[i % len(refs)]["file"]) for i in range(10)]
t0 = time.perf_counter()
model = WhisperModel(opts.get("model", "large-v3-turbo"), device=device, compute_type=ctype,
cpu_threads=threads if device == "cpu" else 0,
download_root=str(work / "models" / "stt"))
transcribe(model, clips[0], 1) # warm-up
res["load_s"] = round(time.perf_counter() - t0, 2)
except Exception as e:
write_json(out, dict(res, status="failed", errors=[f"load: {type(e).__name__}: {str(e)[:300]}"]))
return 0
write_json(out, dict(res, status="loaded"))
res.update(load_loop(lambda: batch_transcribe(model, clips), lambda: transcribe(model, clips[0], 1),
Path(opts["go"]), Path(opts["stop"]), pace=parse_pace(opts)))
write_json(out, res)
return 0
def main() -> int:
out, work, opts = stage_args()
if opts.get("mode") == "load":
return load_worker(out, work, opts)
r = StageResult("stt", out)
r["cuda_dll_dirs"] = len(add_cuda_dll_dirs())
os.environ.setdefault("HF_HOME", str(work / "models" / "hf"))
try:
from faster_whisper import WhisperModel
import ctranslate2
r["versions"] = {"ctranslate2": ctranslate2.__version__,
"faster_whisper": __import__("faster_whisper").__version__,
"cuda_devices": ctranslate2.get_cuda_device_count()}
except Exception as e:
r.error("import", e)
r.save("failed")
return 0
refs = json.loads((ASSETS / "refs.json").read_text(encoding="utf-8"))["samples"]
audio = {s["file"]: read_wav(ASSETS / s["file"]) for s in refs}
name = opts.get("model", "large-v3-turbo")
r["model"] = name
r["audio_source"] = "synthetic MeloTTS Korean (clean speech; real voices will be harder)"
for ctype in opts.get("compute_types", "int8_float16,float16").split(","):
res: dict = {}
r[f"run_{ctype}"] = res
say(f" - {name} / {ctype}")
try:
before = gpu_memory_mib()
t0 = time.perf_counter()
model = WhisperModel(name, device="cuda", compute_type=ctype, download_root=str(work / "models" / "stt"))
res["load_s"] = round(time.perf_counter() - t0, 2)
transcribe(model, audio[refs[0]["file"]], 1) # warm-up (CUDA kernels)
after = gpu_memory_mib()
if before and after:
res["vram_model_mib"] = after[0] - before[0]
say(f" 로딩 {res['load_s']}s, VRAM 약 {res.get('vram_model_mib')} MiB")
except Exception as e:
r.error(f"load {ctype}", e)
continue
# Final transcription, fast (beam 1) and accurate (beam 3).
for beam in (1, 3):
rows = []
for s in refs:
text, sec = transcribe(model, audio[s["file"]], beam)
rows.append({"file": s["file"], "audio_s": s["seconds"], "seconds": round(sec, 3),
"cer": cer(s["text"], text), "text": text})
res[f"final_beam{beam}"] = {"latency": stats(x["seconds"] for x in rows),
"cer": stats([x["cer"] for x in rows], 4), "rows": rows}
say(f" 최종 전사 beam {beam}: 중앙값 {res[f'final_beam{beam}']['latency']['median']}s, "
f"CER {res[f'final_beam{beam}']['cer']['median']}")
# Wake-word hint.
rows = []
for s in refs:
if "자비스" in s["text"]:
plain, _ = transcribe(model, audio[s["file"]], 1)
hinted, _ = transcribe(model, audio[s["file"]], 1, hotwords="자비스")
rows.append({"file": s["file"], "plain": plain, "hotword": hinted,
"plain_has": "자비스" in plain, "hotword_has": "자비스" in hinted})
res["wake_word"] = rows
# Streaming: re-transcribe the growing buffer every 0.5 s (beam 1).
long = max(refs, key=lambda s: s["seconds"])
a = audio[long["file"]]
lat = []
for end in range(SR // 2, len(a) + 1, SR // 2):
_, sec = transcribe(model, a[:end], 1)
lat.append(round(sec, 3))
res["partial_every_0_5s"] = {"audio_s": long["seconds"], "latency": stats(lat), "all": lat}
# The last piece only (what v2 decodes after the end of speech).
_, sec = transcribe(model, a[-SR * 2:], 1)
res["last_2s_piece_s"] = round(sec, 3)
# One shared model for several speakers at once.
batch = {}
with GpuSampler(interval=0.2) as g:
for n in (1, 4, 10):
clips = [audio[refs[i % len(refs)]["file"]] for i in range(n)]
try:
batch_transcribe(model, clips) # warm-up for this batch size
texts, sec = batch_transcribe(model, clips)
errs = [cer(refs[i % len(refs)]["text"], t) for i, t in enumerate(texts)]
batch[str(n)] = {"seconds": round(sec, 3), "cer": stats(errs, 4), "sample_text": texts[0]}
say(f" {n}명 동시(배치): {batch[str(n)]['seconds']}s")
except Exception as e:
r.error(f"batch {n}", e)
batch.update(g.summary())
res["batch_speakers"] = batch
del model
r.save()
r.save("ok" if not r["errors"] else "partial")
return 0
if __name__ == "__main__":
sys.exit(main())