"""Stage: faster-whisper large-v3-turbo speed on the GPU. Runs inside its own venv (faster-whisper, nvidia-cublas-cu12, nvidia-cudnn-cu12). Audio: the synthetic Korean samples in assets/ko_samples (MeloTTS voice). With --opt mode=load it is instead the STT load the gamemode stages put next to the LLM: load the model on device=cuda|cpu, write {"status": "loaded"} to --out, wait for the file go=PATH, then transcribe ten speakers at once (and one final clip) on a loop until the file stop=PATH appears, and write the timings. partial_every=SECONDS|none and final_every=SECONDS pace that loop the way the app would while the LLM answers: a ten-speaker partial pass every N seconds (none: no partials) and a final clip every M seconds in between. """ from __future__ import annotations import glob import json import os import sys import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from m0lib.common import (GpuSampler, StageResult, cer, gpu_memory_mib, say, stage_args, stats, # noqa: E402 write_json) SR = 16000 ASSETS = Path(__file__).resolve().parent.parent / "assets" / "ko_samples" def add_cuda_dll_dirs() -> list[str]: """ctranslate2 loads cuBLAS/cuDNN from the PATH on Windows; the pip wheels put them in nvidia/*/bin.""" found = [] for sp in sys.path: for d in glob.glob(os.path.join(sp, "nvidia", "*", "bin")): if os.path.isdir(d) and d not in found: found.append(d) for d in found: os.environ["PATH"] = d + os.pathsep + os.environ.get("PATH", "") if hasattr(os, "add_dll_directory"): os.add_dll_directory(d) return found def read_wav(path: Path): """The kit's samples are 16 kHz mono PCM16. Read them with the standard library: faster-whisper's PyAV path broke on Windows with a newer PyAV (open() got an unexpected keyword argument 'metadata_errors').""" import wave import numpy as np with wave.open(str(path), "rb") as w: assert w.getframerate() == SR and w.getnchannels() == 1 and w.getsampwidth() == 2, path return np.frombuffer(w.readframes(w.getnframes()), dtype=" tuple[str, float]: t0 = time.perf_counter() segs, _ = model.transcribe(audio, language="ko", beam_size=beam, vad_filter=False, condition_on_previous_text=False, without_timestamps=True, hotwords=hotwords) text = "".join(s.text for s in segs).strip() return text, time.perf_counter() - t0 def batch_transcribe(model, audios: list, beam: int = 1) -> tuple[list[str], float]: """One encoder + decoder pass for several speakers' clips (the v2 design: one shared model).""" import numpy as np from faster_whisper.tokenizer import Tokenizer t0 = time.perf_counter() feats = [] for a in audios: a30 = np.pad(a[: SR * 30], (0, max(0, SR * 30 - len(a)))) feats.append(model.feature_extractor(a30, padding=0)[:, :3000]) enc = model.encode(np.stack(feats).astype(np.float32)) tok = Tokenizer(model.hf_tokenizer, model.model.is_multilingual, task="transcribe", language="ko") prompt = model.get_prompt(tok, previous_tokens=[], without_timestamps=True) res = model.model.generate(enc, [prompt] * len(audios), beam_size=beam, max_length=224, suppress_blank=True, suppress_tokens=[-1]) texts = [tok.decode(x.sequences_ids[0]).strip() for x in res] return texts, time.perf_counter() - t0 def load_loop(batch, single, go: Path, stop: Path, pace: dict | None = None, wait_s: float = 3600, max_s: float = 600) -> dict: """Wait for ``go``, then run ``batch()`` (a ten-speaker partial pass) and ``single()`` (one final clip) until ``stop`` appears. Without ``pace`` they run back to back (every speaker talking without a break). With ``pace`` a partial pass starts every ``partial_every`` s (None: none) and a final every ``final_every`` s. A final that is due goes first; one that falls due during a partial pass waits for it (final_wait_s). An error (out of memory) ends the loop and is kept.""" t0 = time.monotonic() while not go.exists(): if stop.exists() or time.monotonic() - t0 > wait_s: return {"status": "not_started", "batch10_s": stats([]), "single_s": stats([]), "errors": [], "pace": pace} time.sleep(0.2) # perf_counter: time.monotonic ticks every ~16 ms on Windows, too coarse for a 0.1 s clip. b, one, waits, errors = [], [], [], [] t1 = time.perf_counter() every_p = (pace or {}).get("partial_every") every_f = (pace or {}).get("final_every") next_p, next_f = t1, t1 def timed(fn, into): s0 = time.perf_counter() fn() into.append(time.perf_counter() - s0) while not stop.exists() and time.perf_counter() - t1 < max_s: try: if pace is None: timed(batch, b) timed(single, one) continue now = time.perf_counter() # A final goes first: the answer waits for it, not for the subtitles. if every_f is not None and now >= next_f: waits.append(now - next_f) next_f = max(next_f + every_f, now) timed(single, one) continue if every_p is not None and now >= next_p: next_p = now + every_p timed(batch, b) continue dues = [x for x, on in ((next_p, every_p is not None), (next_f, every_f is not None)) if on] time.sleep(min(max(min(dues, default=now + 0.1) - now, 0.0), 0.1)) except Exception as e: errors.append(f"{type(e).__name__}: {str(e)[:300]}") break seconds = time.perf_counter() - t1 res = {"status": "done", "batch10_s": stats(b), "single_s": stats(one), "errors": errors, "seconds": round(seconds, 1), "pace": pace, "busy_share": round(min((sum(b) + sum(one)) / seconds, 1.0), 3) if seconds > 0 else None} if pace is not None: res["final_wait_s"] = stats(waits) return res def parse_pace(opts: dict) -> dict | None: """partial_every=SECONDS|none, final_every=SECONDS; neither: back to back (None).""" if "partial_every" not in opts and "final_every" not in opts: return None def num(key): v = opts.get(key, "none") return None if v == "none" else float(v) return {"partial_every": num("partial_every"), "final_every": num("final_every")} def load_worker(out: Path, work: Path, opts: dict) -> int: device = opts.get("device", "cuda") ctype = opts.get("compute", "int8_float16" if device == "cuda" else "int8") threads = int(opts.get("threads", "8")) res: dict = {"device": device, "compute_type": ctype, "cpu_threads": threads if device == "cpu" else None} try: if device == "cuda": add_cuda_dll_dirs() from faster_whisper import WhisperModel refs = json.loads((ASSETS / "refs.json").read_text(encoding="utf-8"))["samples"] clips = [read_wav(ASSETS / refs[i % len(refs)]["file"]) for i in range(10)] t0 = time.perf_counter() model = WhisperModel(opts.get("model", "large-v3-turbo"), device=device, compute_type=ctype, cpu_threads=threads if device == "cpu" else 0, download_root=str(work / "models" / "stt")) transcribe(model, clips[0], 1) # warm-up res["load_s"] = round(time.perf_counter() - t0, 2) except Exception as e: write_json(out, dict(res, status="failed", errors=[f"load: {type(e).__name__}: {str(e)[:300]}"])) return 0 write_json(out, dict(res, status="loaded")) res.update(load_loop(lambda: batch_transcribe(model, clips), lambda: transcribe(model, clips[0], 1), Path(opts["go"]), Path(opts["stop"]), pace=parse_pace(opts))) write_json(out, res) return 0 def main() -> int: out, work, opts = stage_args() if opts.get("mode") == "load": return load_worker(out, work, opts) r = StageResult("stt", out) r["cuda_dll_dirs"] = len(add_cuda_dll_dirs()) os.environ.setdefault("HF_HOME", str(work / "models" / "hf")) try: from faster_whisper import WhisperModel import ctranslate2 r["versions"] = {"ctranslate2": ctranslate2.__version__, "faster_whisper": __import__("faster_whisper").__version__, "cuda_devices": ctranslate2.get_cuda_device_count()} except Exception as e: r.error("import", e) r.save("failed") return 0 refs = json.loads((ASSETS / "refs.json").read_text(encoding="utf-8"))["samples"] audio = {s["file"]: read_wav(ASSETS / s["file"]) for s in refs} name = opts.get("model", "large-v3-turbo") r["model"] = name r["audio_source"] = "synthetic MeloTTS Korean (clean speech; real voices will be harder)" for ctype in opts.get("compute_types", "int8_float16,float16").split(","): res: dict = {} r[f"run_{ctype}"] = res say(f" - {name} / {ctype}") try: before = gpu_memory_mib() t0 = time.perf_counter() model = WhisperModel(name, device="cuda", compute_type=ctype, download_root=str(work / "models" / "stt")) res["load_s"] = round(time.perf_counter() - t0, 2) transcribe(model, audio[refs[0]["file"]], 1) # warm-up (CUDA kernels) after = gpu_memory_mib() if before and after: res["vram_model_mib"] = after[0] - before[0] say(f" 로딩 {res['load_s']}s, VRAM 약 {res.get('vram_model_mib')} MiB") except Exception as e: r.error(f"load {ctype}", e) continue # Final transcription, fast (beam 1) and accurate (beam 3). for beam in (1, 3): rows = [] for s in refs: text, sec = transcribe(model, audio[s["file"]], beam) rows.append({"file": s["file"], "audio_s": s["seconds"], "seconds": round(sec, 3), "cer": cer(s["text"], text), "text": text}) res[f"final_beam{beam}"] = {"latency": stats(x["seconds"] for x in rows), "cer": stats([x["cer"] for x in rows], 4), "rows": rows} say(f" 최종 전사 beam {beam}: 중앙값 {res[f'final_beam{beam}']['latency']['median']}s, " f"CER {res[f'final_beam{beam}']['cer']['median']}") # Wake-word hint. rows = [] for s in refs: if "자비스" in s["text"]: plain, _ = transcribe(model, audio[s["file"]], 1) hinted, _ = transcribe(model, audio[s["file"]], 1, hotwords="자비스") rows.append({"file": s["file"], "plain": plain, "hotword": hinted, "plain_has": "자비스" in plain, "hotword_has": "자비스" in hinted}) res["wake_word"] = rows # Streaming: re-transcribe the growing buffer every 0.5 s (beam 1). long = max(refs, key=lambda s: s["seconds"]) a = audio[long["file"]] lat = [] for end in range(SR // 2, len(a) + 1, SR // 2): _, sec = transcribe(model, a[:end], 1) lat.append(round(sec, 3)) res["partial_every_0_5s"] = {"audio_s": long["seconds"], "latency": stats(lat), "all": lat} # The last piece only (what v2 decodes after the end of speech). _, sec = transcribe(model, a[-SR * 2:], 1) res["last_2s_piece_s"] = round(sec, 3) # One shared model for several speakers at once. batch = {} with GpuSampler(interval=0.2) as g: for n in (1, 4, 10): clips = [audio[refs[i % len(refs)]["file"]] for i in range(n)] try: batch_transcribe(model, clips) # warm-up for this batch size texts, sec = batch_transcribe(model, clips) errs = [cer(refs[i % len(refs)]["text"], t) for i, t in enumerate(texts)] batch[str(n)] = {"seconds": round(sec, 3), "cer": stats(errs, 4), "sample_text": texts[0]} say(f" {n}명 동시(배치): {batch[str(n)]['seconds']}s") except Exception as e: r.error(f"batch {n}", e) batch.update(g.summary()) res["batch_speakers"] = batch del model r.save() r.save("ok" if not r["errors"] else "partial") return 0 if __name__ == "__main__": sys.exit(main())