perf(voice): run STT+TTS on the GPU by default with CPU fallback
Both voice backends defaulted to CPU. Fix the "CUDA unavailable" gaps so everything that benefits from the RTX 5050 uses it: - MeloTTS venv had CPU-only torch (2.12.0+cpu) -> installed Blackwell-capable torch/torchaudio 2.11.0+cu128 (sm_120 verified with a real GPU matmul). - faster-whisper CUDA loaded but transcribe() died with "libcublas.so.12 not found": installed nvidia-cublas-cu12 + nvidia-cudnn-cu12 into the whisper venv and inject those nvidia/*/lib dirs into the worker's LD_LIBRARY_PATH at spawn (the loader only honours it at exec). - WSAI_WHISPER_DEVICE / WSAI_MELO_DEVICE now default to "auto": pick CUDA when present, else CPU, and each worker falls back to CPU if a CUDA load fails so the voice loop never dies on a GPU-less host. Verified end-to-end through the real backend classes: both workers report "ready on cuda"; steady-state STT ~170ms (was ~1350ms CPU), TTS ~4s first call vs ~23s CPU. All 12 tests pass. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -39,17 +39,42 @@ def _log(*a):
|
||||
|
||||
def main() -> None:
|
||||
model_name = os.environ.get("WSAI_WHISPER_MODEL", "small")
|
||||
device = os.environ.get("WSAI_WHISPER_DEVICE", "cpu") # "cpu" | "cuda"
|
||||
# int8 on CPU keeps a small model fast; float16 is the usual CUDA choice.
|
||||
compute = os.environ.get(
|
||||
"WSAI_WHISPER_COMPUTE", "int8" if device == "cpu" else "float16"
|
||||
)
|
||||
requested = os.environ.get("WSAI_WHISPER_DEVICE", "auto") # cpu | cuda | auto
|
||||
default_lang = os.environ.get("WSAI_WHISPER_LANGUAGE", "ko") or None
|
||||
|
||||
t0 = time.monotonic()
|
||||
from faster_whisper import WhisperModel # heavy import; only in whisper venv
|
||||
|
||||
model = WhisperModel(model_name, device=device, compute_type=compute)
|
||||
def _has_cuda() -> bool:
|
||||
try:
|
||||
import ctranslate2
|
||||
|
||||
return ctranslate2.get_cuda_device_count() > 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
device = requested
|
||||
if requested == "auto":
|
||||
device = "cuda" if _has_cuda() else "cpu"
|
||||
|
||||
def _compute_for(dev: str) -> str:
|
||||
# int8 on CPU keeps a small model fast; float16 is the usual CUDA choice.
|
||||
return os.environ.get(
|
||||
"WSAI_WHISPER_COMPUTE", "int8" if dev == "cpu" else "float16"
|
||||
)
|
||||
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
model = WhisperModel(model_name, device=device, compute_type=_compute_for(device))
|
||||
except Exception as exc:
|
||||
# CUDA picked but unusable (missing libs, OOM): fall back to CPU rather
|
||||
# than leaving the whole voice loop dead.
|
||||
if device == "cuda":
|
||||
_log(f"[whisper_worker] CUDA load failed ({exc}); falling back to CPU")
|
||||
device = "cpu"
|
||||
model = WhisperModel(model_name, device=device, compute_type=_compute_for(device))
|
||||
else:
|
||||
raise
|
||||
compute = _compute_for(device)
|
||||
load_ms = int((time.monotonic() - t0) * 1000)
|
||||
_emit({"ready": True, "ms": load_ms, "device": device, "model": model_name})
|
||||
_log(f"[whisper_worker] {model_name} ready in {load_ms} ms on {device}/{compute}")
|
||||
|
||||
Reference in New Issue
Block a user