The gamemode_stt run showed Strata's automatic placement filling the card next to the STT (15.9 of 16.3 GB, first token 1.6 -> 7.5 s) and the STT's back-to-back partial passes cutting LLM output to a third. The new gamemode_tune stage measures both fixes the user chose: - stt_room: one start without experts (STT loaded) gives the LLM's base VRAM; --expert-vram gets what is left after the STT's ten-speaker growth (from this PC's stt stage) and a 768 MiB margin. - stt_pace: a ten-speaker partial pass every 2 s, or none, with finals every 0.5 s and finals first; records final wait and STT GPU share. run_m0_gamemode_tune.bat runs it unattended. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
278 lines
12 KiB
Python
278 lines
12 KiB
Python
"""Stage: faster-whisper large-v3-turbo speed on the GPU.
|
|
|
|
Runs inside its own venv (faster-whisper, nvidia-cublas-cu12, nvidia-cudnn-cu12).
|
|
Audio: the synthetic Korean samples in assets/ko_samples (MeloTTS voice).
|
|
|
|
With --opt mode=load it is instead the STT load the gamemode stages put next to
|
|
the LLM: load the model on device=cuda|cpu, write {"status": "loaded"} to --out,
|
|
wait for the file go=PATH, then transcribe ten speakers at once (and one final
|
|
clip) on a loop until the file stop=PATH appears, and write the timings.
|
|
partial_every=SECONDS|none and final_every=SECONDS pace that loop the way the app
|
|
would while the LLM answers: a ten-speaker partial pass every N seconds (none: no
|
|
partials) and a final clip every M seconds in between.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import glob
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
from m0lib.common import (GpuSampler, StageResult, cer, gpu_memory_mib, say, stage_args, stats, # noqa: E402
|
|
write_json)
|
|
|
|
SR = 16000
|
|
ASSETS = Path(__file__).resolve().parent.parent / "assets" / "ko_samples"
|
|
|
|
|
|
def add_cuda_dll_dirs() -> list[str]:
|
|
"""ctranslate2 loads cuBLAS/cuDNN from the PATH on Windows; the pip wheels put them in nvidia/*/bin."""
|
|
found = []
|
|
for sp in sys.path:
|
|
for d in glob.glob(os.path.join(sp, "nvidia", "*", "bin")):
|
|
if os.path.isdir(d) and d not in found:
|
|
found.append(d)
|
|
for d in found:
|
|
os.environ["PATH"] = d + os.pathsep + os.environ.get("PATH", "")
|
|
if hasattr(os, "add_dll_directory"):
|
|
os.add_dll_directory(d)
|
|
return found
|
|
|
|
|
|
def read_wav(path: Path):
|
|
"""The kit's samples are 16 kHz mono PCM16. Read them with the standard library:
|
|
faster-whisper's PyAV path broke on Windows with a newer PyAV
|
|
(open() got an unexpected keyword argument 'metadata_errors')."""
|
|
import wave
|
|
import numpy as np
|
|
with wave.open(str(path), "rb") as w:
|
|
assert w.getframerate() == SR and w.getnchannels() == 1 and w.getsampwidth() == 2, path
|
|
return np.frombuffer(w.readframes(w.getnframes()), dtype="<i2").astype(np.float32) / 32768.0
|
|
|
|
|
|
def transcribe(model, audio, beam: int, hotwords: str | None = None) -> tuple[str, float]:
|
|
t0 = time.perf_counter()
|
|
segs, _ = model.transcribe(audio, language="ko", beam_size=beam, vad_filter=False,
|
|
condition_on_previous_text=False, without_timestamps=True, hotwords=hotwords)
|
|
text = "".join(s.text for s in segs).strip()
|
|
return text, time.perf_counter() - t0
|
|
|
|
|
|
def batch_transcribe(model, audios: list, beam: int = 1) -> tuple[list[str], float]:
|
|
"""One encoder + decoder pass for several speakers' clips (the v2 design: one shared model)."""
|
|
import numpy as np
|
|
from faster_whisper.tokenizer import Tokenizer
|
|
t0 = time.perf_counter()
|
|
feats = []
|
|
for a in audios:
|
|
a30 = np.pad(a[: SR * 30], (0, max(0, SR * 30 - len(a))))
|
|
feats.append(model.feature_extractor(a30, padding=0)[:, :3000])
|
|
enc = model.encode(np.stack(feats).astype(np.float32))
|
|
tok = Tokenizer(model.hf_tokenizer, model.model.is_multilingual, task="transcribe", language="ko")
|
|
prompt = model.get_prompt(tok, previous_tokens=[], without_timestamps=True)
|
|
res = model.model.generate(enc, [prompt] * len(audios), beam_size=beam, max_length=224, suppress_blank=True,
|
|
suppress_tokens=[-1])
|
|
texts = [tok.decode(x.sequences_ids[0]).strip() for x in res]
|
|
return texts, time.perf_counter() - t0
|
|
|
|
|
|
def load_loop(batch, single, go: Path, stop: Path, pace: dict | None = None, wait_s: float = 3600,
|
|
max_s: float = 600) -> dict:
|
|
"""Wait for ``go``, then run ``batch()`` (a ten-speaker partial pass) and ``single()`` (one final
|
|
clip) until ``stop`` appears. Without ``pace`` they run back to back (every speaker talking
|
|
without a break). With ``pace`` a partial pass starts every ``partial_every`` s (None: none) and a
|
|
final every ``final_every`` s. A final that is due goes first; one that falls due during a
|
|
partial pass waits for it (final_wait_s).
|
|
An error (out of memory) ends the loop and is kept."""
|
|
t0 = time.monotonic()
|
|
while not go.exists():
|
|
if stop.exists() or time.monotonic() - t0 > wait_s:
|
|
return {"status": "not_started", "batch10_s": stats([]), "single_s": stats([]), "errors": [],
|
|
"pace": pace}
|
|
time.sleep(0.2)
|
|
# perf_counter: time.monotonic ticks every ~16 ms on Windows, too coarse for a 0.1 s clip.
|
|
b, one, waits, errors = [], [], [], []
|
|
t1 = time.perf_counter()
|
|
every_p = (pace or {}).get("partial_every")
|
|
every_f = (pace or {}).get("final_every")
|
|
next_p, next_f = t1, t1
|
|
|
|
def timed(fn, into):
|
|
s0 = time.perf_counter()
|
|
fn()
|
|
into.append(time.perf_counter() - s0)
|
|
|
|
while not stop.exists() and time.perf_counter() - t1 < max_s:
|
|
try:
|
|
if pace is None:
|
|
timed(batch, b)
|
|
timed(single, one)
|
|
continue
|
|
now = time.perf_counter()
|
|
# A final goes first: the answer waits for it, not for the subtitles.
|
|
if every_f is not None and now >= next_f:
|
|
waits.append(now - next_f)
|
|
next_f = max(next_f + every_f, now)
|
|
timed(single, one)
|
|
continue
|
|
if every_p is not None and now >= next_p:
|
|
next_p = now + every_p
|
|
timed(batch, b)
|
|
continue
|
|
dues = [x for x, on in ((next_p, every_p is not None), (next_f, every_f is not None)) if on]
|
|
time.sleep(min(max(min(dues, default=now + 0.1) - now, 0.0), 0.1))
|
|
except Exception as e:
|
|
errors.append(f"{type(e).__name__}: {str(e)[:300]}")
|
|
break
|
|
seconds = time.perf_counter() - t1
|
|
res = {"status": "done", "batch10_s": stats(b), "single_s": stats(one), "errors": errors,
|
|
"seconds": round(seconds, 1), "pace": pace,
|
|
"busy_share": round(min((sum(b) + sum(one)) / seconds, 1.0), 3) if seconds > 0 else None}
|
|
if pace is not None:
|
|
res["final_wait_s"] = stats(waits)
|
|
return res
|
|
|
|
|
|
def parse_pace(opts: dict) -> dict | None:
|
|
"""partial_every=SECONDS|none, final_every=SECONDS; neither: back to back (None)."""
|
|
if "partial_every" not in opts and "final_every" not in opts:
|
|
return None
|
|
|
|
def num(key):
|
|
v = opts.get(key, "none")
|
|
return None if v == "none" else float(v)
|
|
|
|
return {"partial_every": num("partial_every"), "final_every": num("final_every")}
|
|
|
|
|
|
def load_worker(out: Path, work: Path, opts: dict) -> int:
|
|
device = opts.get("device", "cuda")
|
|
ctype = opts.get("compute", "int8_float16" if device == "cuda" else "int8")
|
|
threads = int(opts.get("threads", "8"))
|
|
res: dict = {"device": device, "compute_type": ctype, "cpu_threads": threads if device == "cpu" else None}
|
|
try:
|
|
if device == "cuda":
|
|
add_cuda_dll_dirs()
|
|
from faster_whisper import WhisperModel
|
|
refs = json.loads((ASSETS / "refs.json").read_text(encoding="utf-8"))["samples"]
|
|
clips = [read_wav(ASSETS / refs[i % len(refs)]["file"]) for i in range(10)]
|
|
t0 = time.perf_counter()
|
|
model = WhisperModel(opts.get("model", "large-v3-turbo"), device=device, compute_type=ctype,
|
|
cpu_threads=threads if device == "cpu" else 0,
|
|
download_root=str(work / "models" / "stt"))
|
|
transcribe(model, clips[0], 1) # warm-up
|
|
res["load_s"] = round(time.perf_counter() - t0, 2)
|
|
except Exception as e:
|
|
write_json(out, dict(res, status="failed", errors=[f"load: {type(e).__name__}: {str(e)[:300]}"]))
|
|
return 0
|
|
write_json(out, dict(res, status="loaded"))
|
|
res.update(load_loop(lambda: batch_transcribe(model, clips), lambda: transcribe(model, clips[0], 1),
|
|
Path(opts["go"]), Path(opts["stop"]), pace=parse_pace(opts)))
|
|
write_json(out, res)
|
|
return 0
|
|
|
|
|
|
def main() -> int:
|
|
out, work, opts = stage_args()
|
|
if opts.get("mode") == "load":
|
|
return load_worker(out, work, opts)
|
|
r = StageResult("stt", out)
|
|
r["cuda_dll_dirs"] = len(add_cuda_dll_dirs())
|
|
os.environ.setdefault("HF_HOME", str(work / "models" / "hf"))
|
|
try:
|
|
from faster_whisper import WhisperModel
|
|
import ctranslate2
|
|
r["versions"] = {"ctranslate2": ctranslate2.__version__,
|
|
"faster_whisper": __import__("faster_whisper").__version__,
|
|
"cuda_devices": ctranslate2.get_cuda_device_count()}
|
|
except Exception as e:
|
|
r.error("import", e)
|
|
r.save("failed")
|
|
return 0
|
|
|
|
refs = json.loads((ASSETS / "refs.json").read_text(encoding="utf-8"))["samples"]
|
|
audio = {s["file"]: read_wav(ASSETS / s["file"]) for s in refs}
|
|
name = opts.get("model", "large-v3-turbo")
|
|
r["model"] = name
|
|
r["audio_source"] = "synthetic MeloTTS Korean (clean speech; real voices will be harder)"
|
|
|
|
for ctype in opts.get("compute_types", "int8_float16,float16").split(","):
|
|
res: dict = {}
|
|
r[f"run_{ctype}"] = res
|
|
say(f" - {name} / {ctype}")
|
|
try:
|
|
before = gpu_memory_mib()
|
|
t0 = time.perf_counter()
|
|
model = WhisperModel(name, device="cuda", compute_type=ctype, download_root=str(work / "models" / "stt"))
|
|
res["load_s"] = round(time.perf_counter() - t0, 2)
|
|
transcribe(model, audio[refs[0]["file"]], 1) # warm-up (CUDA kernels)
|
|
after = gpu_memory_mib()
|
|
if before and after:
|
|
res["vram_model_mib"] = after[0] - before[0]
|
|
say(f" 로딩 {res['load_s']}s, VRAM 약 {res.get('vram_model_mib')} MiB")
|
|
except Exception as e:
|
|
r.error(f"load {ctype}", e)
|
|
continue
|
|
|
|
# Final transcription, fast (beam 1) and accurate (beam 3).
|
|
for beam in (1, 3):
|
|
rows = []
|
|
for s in refs:
|
|
text, sec = transcribe(model, audio[s["file"]], beam)
|
|
rows.append({"file": s["file"], "audio_s": s["seconds"], "seconds": round(sec, 3),
|
|
"cer": cer(s["text"], text), "text": text})
|
|
res[f"final_beam{beam}"] = {"latency": stats(x["seconds"] for x in rows),
|
|
"cer": stats([x["cer"] for x in rows], 4), "rows": rows}
|
|
say(f" 최종 전사 beam {beam}: 중앙값 {res[f'final_beam{beam}']['latency']['median']}s, "
|
|
f"CER {res[f'final_beam{beam}']['cer']['median']}")
|
|
|
|
# Wake-word hint.
|
|
rows = []
|
|
for s in refs:
|
|
if "자비스" in s["text"]:
|
|
plain, _ = transcribe(model, audio[s["file"]], 1)
|
|
hinted, _ = transcribe(model, audio[s["file"]], 1, hotwords="자비스")
|
|
rows.append({"file": s["file"], "plain": plain, "hotword": hinted,
|
|
"plain_has": "자비스" in plain, "hotword_has": "자비스" in hinted})
|
|
res["wake_word"] = rows
|
|
|
|
# Streaming: re-transcribe the growing buffer every 0.5 s (beam 1).
|
|
long = max(refs, key=lambda s: s["seconds"])
|
|
a = audio[long["file"]]
|
|
lat = []
|
|
for end in range(SR // 2, len(a) + 1, SR // 2):
|
|
_, sec = transcribe(model, a[:end], 1)
|
|
lat.append(round(sec, 3))
|
|
res["partial_every_0_5s"] = {"audio_s": long["seconds"], "latency": stats(lat), "all": lat}
|
|
# The last piece only (what v2 decodes after the end of speech).
|
|
_, sec = transcribe(model, a[-SR * 2:], 1)
|
|
res["last_2s_piece_s"] = round(sec, 3)
|
|
|
|
# One shared model for several speakers at once.
|
|
batch = {}
|
|
with GpuSampler(interval=0.2) as g:
|
|
for n in (1, 4, 10):
|
|
clips = [audio[refs[i % len(refs)]["file"]] for i in range(n)]
|
|
try:
|
|
batch_transcribe(model, clips) # warm-up for this batch size
|
|
texts, sec = batch_transcribe(model, clips)
|
|
errs = [cer(refs[i % len(refs)]["text"], t) for i, t in enumerate(texts)]
|
|
batch[str(n)] = {"seconds": round(sec, 3), "cer": stats(errs, 4), "sample_text": texts[0]}
|
|
say(f" {n}명 동시(배치): {batch[str(n)]['seconds']}s")
|
|
except Exception as e:
|
|
r.error(f"batch {n}", e)
|
|
batch.update(g.summary())
|
|
res["batch_speakers"] = batch
|
|
del model
|
|
r.save()
|
|
r.save("ok" if not r["errors"] else "partial")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|