gamemode now also starts the 2 GB expert placement with a 16K context and drops the --kv q4_0 and --mtp off rows the Strata 0.1.59 qwen36 engine ignores; the normal placement answers one request at a time. gamemode_stt loads the STT worker (stt.py mode=load) before the LLM picks its placement and keeps it transcribing ten speakers at once while the LLM is measured, so the VRAM left is the real game share. The game placements run with the STT on the GPU and on the CPU. gamemode_live repeats the game rows while the user runs a real game; it only runs when asked for (run_m0_game_live.bat) and is skipped when nobody is at the PC. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
229 lines
10 KiB
Python
229 lines
10 KiB
Python
"""Stage: faster-whisper large-v3-turbo speed on the GPU.
|
|
|
|
Runs inside its own venv (faster-whisper, nvidia-cublas-cu12, nvidia-cudnn-cu12).
|
|
Audio: the synthetic Korean samples in assets/ko_samples (MeloTTS voice).
|
|
|
|
With --opt mode=load it is instead the STT load the gamemode stages put next to
|
|
the LLM: load the model on device=cuda|cpu, write {"status": "loaded"} to --out,
|
|
wait for the file go=PATH, then transcribe ten speakers at once (and one final
|
|
clip) on a loop until the file stop=PATH appears, and write the timings.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import glob
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
from m0lib.common import (GpuSampler, StageResult, cer, gpu_memory_mib, say, stage_args, stats, # noqa: E402
|
|
write_json)
|
|
|
|
SR = 16000
|
|
ASSETS = Path(__file__).resolve().parent.parent / "assets" / "ko_samples"
|
|
|
|
|
|
def add_cuda_dll_dirs() -> list[str]:
|
|
"""ctranslate2 loads cuBLAS/cuDNN from the PATH on Windows; the pip wheels put them in nvidia/*/bin."""
|
|
found = []
|
|
for sp in sys.path:
|
|
for d in glob.glob(os.path.join(sp, "nvidia", "*", "bin")):
|
|
if os.path.isdir(d) and d not in found:
|
|
found.append(d)
|
|
for d in found:
|
|
os.environ["PATH"] = d + os.pathsep + os.environ.get("PATH", "")
|
|
if hasattr(os, "add_dll_directory"):
|
|
os.add_dll_directory(d)
|
|
return found
|
|
|
|
|
|
def read_wav(path: Path):
|
|
"""The kit's samples are 16 kHz mono PCM16. Read them with the standard library:
|
|
faster-whisper's PyAV path broke on Windows with a newer PyAV
|
|
(open() got an unexpected keyword argument 'metadata_errors')."""
|
|
import wave
|
|
import numpy as np
|
|
with wave.open(str(path), "rb") as w:
|
|
assert w.getframerate() == SR and w.getnchannels() == 1 and w.getsampwidth() == 2, path
|
|
return np.frombuffer(w.readframes(w.getnframes()), dtype="<i2").astype(np.float32) / 32768.0
|
|
|
|
|
|
def transcribe(model, audio, beam: int, hotwords: str | None = None) -> tuple[str, float]:
|
|
t0 = time.perf_counter()
|
|
segs, _ = model.transcribe(audio, language="ko", beam_size=beam, vad_filter=False,
|
|
condition_on_previous_text=False, without_timestamps=True, hotwords=hotwords)
|
|
text = "".join(s.text for s in segs).strip()
|
|
return text, time.perf_counter() - t0
|
|
|
|
|
|
def batch_transcribe(model, audios: list, beam: int = 1) -> tuple[list[str], float]:
|
|
"""One encoder + decoder pass for several speakers' clips (the v2 design: one shared model)."""
|
|
import numpy as np
|
|
from faster_whisper.tokenizer import Tokenizer
|
|
t0 = time.perf_counter()
|
|
feats = []
|
|
for a in audios:
|
|
a30 = np.pad(a[: SR * 30], (0, max(0, SR * 30 - len(a))))
|
|
feats.append(model.feature_extractor(a30, padding=0)[:, :3000])
|
|
enc = model.encode(np.stack(feats).astype(np.float32))
|
|
tok = Tokenizer(model.hf_tokenizer, model.model.is_multilingual, task="transcribe", language="ko")
|
|
prompt = model.get_prompt(tok, previous_tokens=[], without_timestamps=True)
|
|
res = model.model.generate(enc, [prompt] * len(audios), beam_size=beam, max_length=224, suppress_blank=True,
|
|
suppress_tokens=[-1])
|
|
texts = [tok.decode(x.sequences_ids[0]).strip() for x in res]
|
|
return texts, time.perf_counter() - t0
|
|
|
|
|
|
def load_loop(batch, single, go: Path, stop: Path, wait_s: float = 3600, max_s: float = 600) -> dict:
|
|
"""Wait for ``go``, then run ``batch()`` (ten speakers) and ``single()`` (one final clip)
|
|
in turn until ``stop`` appears. An error (out of memory) ends the loop and is kept."""
|
|
t0 = time.monotonic()
|
|
while not go.exists():
|
|
if stop.exists() or time.monotonic() - t0 > wait_s:
|
|
return {"status": "not_started", "batch10_s": stats([]), "single_s": stats([]), "errors": []}
|
|
time.sleep(0.2)
|
|
b, one, errors = [], [], []
|
|
t1 = time.monotonic()
|
|
while not stop.exists() and time.monotonic() - t1 < max_s:
|
|
try:
|
|
for fn, into in ((batch, b), (single, one)):
|
|
s0 = time.perf_counter()
|
|
fn()
|
|
into.append(time.perf_counter() - s0)
|
|
except Exception as e:
|
|
errors.append(f"{type(e).__name__}: {str(e)[:300]}")
|
|
break
|
|
return {"status": "done", "batch10_s": stats(b), "single_s": stats(one), "errors": errors,
|
|
"seconds": round(time.monotonic() - t1, 1)}
|
|
|
|
|
|
def load_worker(out: Path, work: Path, opts: dict) -> int:
|
|
device = opts.get("device", "cuda")
|
|
ctype = opts.get("compute", "int8_float16" if device == "cuda" else "int8")
|
|
threads = int(opts.get("threads", "8"))
|
|
res: dict = {"device": device, "compute_type": ctype, "cpu_threads": threads if device == "cpu" else None}
|
|
try:
|
|
if device == "cuda":
|
|
add_cuda_dll_dirs()
|
|
from faster_whisper import WhisperModel
|
|
refs = json.loads((ASSETS / "refs.json").read_text(encoding="utf-8"))["samples"]
|
|
clips = [read_wav(ASSETS / refs[i % len(refs)]["file"]) for i in range(10)]
|
|
t0 = time.perf_counter()
|
|
model = WhisperModel(opts.get("model", "large-v3-turbo"), device=device, compute_type=ctype,
|
|
cpu_threads=threads if device == "cpu" else 0,
|
|
download_root=str(work / "models" / "stt"))
|
|
transcribe(model, clips[0], 1) # warm-up
|
|
res["load_s"] = round(time.perf_counter() - t0, 2)
|
|
except Exception as e:
|
|
write_json(out, dict(res, status="failed", errors=[f"load: {type(e).__name__}: {str(e)[:300]}"]))
|
|
return 0
|
|
write_json(out, dict(res, status="loaded"))
|
|
res.update(load_loop(lambda: batch_transcribe(model, clips), lambda: transcribe(model, clips[0], 1),
|
|
Path(opts["go"]), Path(opts["stop"])))
|
|
write_json(out, res)
|
|
return 0
|
|
|
|
|
|
def main() -> int:
|
|
out, work, opts = stage_args()
|
|
if opts.get("mode") == "load":
|
|
return load_worker(out, work, opts)
|
|
r = StageResult("stt", out)
|
|
r["cuda_dll_dirs"] = len(add_cuda_dll_dirs())
|
|
os.environ.setdefault("HF_HOME", str(work / "models" / "hf"))
|
|
try:
|
|
from faster_whisper import WhisperModel
|
|
import ctranslate2
|
|
r["versions"] = {"ctranslate2": ctranslate2.__version__,
|
|
"faster_whisper": __import__("faster_whisper").__version__,
|
|
"cuda_devices": ctranslate2.get_cuda_device_count()}
|
|
except Exception as e:
|
|
r.error("import", e)
|
|
r.save("failed")
|
|
return 0
|
|
|
|
refs = json.loads((ASSETS / "refs.json").read_text(encoding="utf-8"))["samples"]
|
|
audio = {s["file"]: read_wav(ASSETS / s["file"]) for s in refs}
|
|
name = opts.get("model", "large-v3-turbo")
|
|
r["model"] = name
|
|
r["audio_source"] = "synthetic MeloTTS Korean (clean speech; real voices will be harder)"
|
|
|
|
for ctype in opts.get("compute_types", "int8_float16,float16").split(","):
|
|
res: dict = {}
|
|
r[f"run_{ctype}"] = res
|
|
say(f" - {name} / {ctype}")
|
|
try:
|
|
before = gpu_memory_mib()
|
|
t0 = time.perf_counter()
|
|
model = WhisperModel(name, device="cuda", compute_type=ctype, download_root=str(work / "models" / "stt"))
|
|
res["load_s"] = round(time.perf_counter() - t0, 2)
|
|
transcribe(model, audio[refs[0]["file"]], 1) # warm-up (CUDA kernels)
|
|
after = gpu_memory_mib()
|
|
if before and after:
|
|
res["vram_model_mib"] = after[0] - before[0]
|
|
say(f" 로딩 {res['load_s']}s, VRAM 약 {res.get('vram_model_mib')} MiB")
|
|
except Exception as e:
|
|
r.error(f"load {ctype}", e)
|
|
continue
|
|
|
|
# Final transcription, fast (beam 1) and accurate (beam 3).
|
|
for beam in (1, 3):
|
|
rows = []
|
|
for s in refs:
|
|
text, sec = transcribe(model, audio[s["file"]], beam)
|
|
rows.append({"file": s["file"], "audio_s": s["seconds"], "seconds": round(sec, 3),
|
|
"cer": cer(s["text"], text), "text": text})
|
|
res[f"final_beam{beam}"] = {"latency": stats(x["seconds"] for x in rows),
|
|
"cer": stats([x["cer"] for x in rows], 4), "rows": rows}
|
|
say(f" 최종 전사 beam {beam}: 중앙값 {res[f'final_beam{beam}']['latency']['median']}s, "
|
|
f"CER {res[f'final_beam{beam}']['cer']['median']}")
|
|
|
|
# Wake-word hint.
|
|
rows = []
|
|
for s in refs:
|
|
if "자비스" in s["text"]:
|
|
plain, _ = transcribe(model, audio[s["file"]], 1)
|
|
hinted, _ = transcribe(model, audio[s["file"]], 1, hotwords="자비스")
|
|
rows.append({"file": s["file"], "plain": plain, "hotword": hinted,
|
|
"plain_has": "자비스" in plain, "hotword_has": "자비스" in hinted})
|
|
res["wake_word"] = rows
|
|
|
|
# Streaming: re-transcribe the growing buffer every 0.5 s (beam 1).
|
|
long = max(refs, key=lambda s: s["seconds"])
|
|
a = audio[long["file"]]
|
|
lat = []
|
|
for end in range(SR // 2, len(a) + 1, SR // 2):
|
|
_, sec = transcribe(model, a[:end], 1)
|
|
lat.append(round(sec, 3))
|
|
res["partial_every_0_5s"] = {"audio_s": long["seconds"], "latency": stats(lat), "all": lat}
|
|
# The last piece only (what v2 decodes after the end of speech).
|
|
_, sec = transcribe(model, a[-SR * 2:], 1)
|
|
res["last_2s_piece_s"] = round(sec, 3)
|
|
|
|
# One shared model for several speakers at once.
|
|
batch = {}
|
|
with GpuSampler(interval=0.2) as g:
|
|
for n in (1, 4, 10):
|
|
clips = [audio[refs[i % len(refs)]["file"]] for i in range(n)]
|
|
try:
|
|
batch_transcribe(model, clips) # warm-up for this batch size
|
|
texts, sec = batch_transcribe(model, clips)
|
|
errs = [cer(refs[i % len(refs)]["text"], t) for i, t in enumerate(texts)]
|
|
batch[str(n)] = {"seconds": round(sec, 3), "cer": stats(errs, 4), "sample_text": texts[0]}
|
|
say(f" {n}명 동시(배치): {batch[str(n)]['seconds']}s")
|
|
except Exception as e:
|
|
r.error(f"batch {n}", e)
|
|
batch.update(g.summary())
|
|
res["batch_speakers"] = batch
|
|
del model
|
|
r.save()
|
|
r.save("ok" if not r["errors"] else "partial")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|