fix: serialize worker warmup handshake to stop concurrent stdout reads
The STT/TTS worker _ensure() treated a spawned-but-not-yet-handshaked subprocess as ready, so a voice turn arriving during warmup read the same stdout StreamReader concurrently with the warmup handshake and crashed with "readuntil() called while another coroutine is already waiting for incoming data". Add a _start_lock + _ready flag so (re)start and the ready handshake run atomically and callers wait for real readiness before reading stdout. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -75,6 +75,11 @@ class WhisperSTT:
|
||||
self.audio_source = audio_source
|
||||
self._proc: asyncio.subprocess.Process | None = None
|
||||
self._lock = asyncio.Lock()
|
||||
# Serialises worker (re)start + the ready handshake so a caller that
|
||||
# arrives mid-warmup waits for readiness instead of reading the same
|
||||
# stdout StreamReader concurrently (asyncio forbids overlapping reads).
|
||||
self._start_lock = asyncio.Lock()
|
||||
self._ready = False # True only after the ready handshake completes
|
||||
self.load_ms: int | None = None
|
||||
self.resolved_device: str | None = None # "cuda" | "cpu", known after start
|
||||
# Keep the worker's most recent stderr so a crash reports its real cause
|
||||
@@ -106,59 +111,70 @@ class WhisperSTT:
|
||||
await self._ensure()
|
||||
|
||||
async def _ensure(self) -> None:
|
||||
if self._proc is not None and self._proc.returncode is None:
|
||||
# Fast path: only skip when the worker is not just spawned but fully
|
||||
# handshaked. Checking `_proc` alone would let a caller sail past while
|
||||
# another coroutine (e.g. warmup) is still awaiting the ready line on
|
||||
# this same stdout, causing overlapping StreamReader reads.
|
||||
if self._proc is not None and self._proc.returncode is None and self._ready:
|
||||
return
|
||||
env = {
|
||||
**os.environ,
|
||||
"WSAI_WHISPER_MODEL": self.model,
|
||||
"WSAI_WHISPER_DEVICE": self.device,
|
||||
}
|
||||
# ctranslate2 dlopens libcublas/libcudnn from the whisper venv's nvidia
|
||||
# pip packages; the dynamic loader only honours LD_LIBRARY_PATH captured
|
||||
# at exec, so inject those lib dirs into the child env here (harmless on
|
||||
# CPU). Without this the CUDA model loads but transcribe() dies with
|
||||
# "Library libcublas.so.12 is not found".
|
||||
lib_dirs = _cuda_lib_dirs(self.python)
|
||||
if lib_dirs:
|
||||
prev = env.get("LD_LIBRARY_PATH", "")
|
||||
env["LD_LIBRARY_PATH"] = ":".join(lib_dirs + ([prev] if prev else []))
|
||||
if self.language:
|
||||
env["WSAI_WHISPER_LANGUAGE"] = self.language
|
||||
repo_root = str(Path(__file__).resolve().parents[2])
|
||||
self._proc = await asyncio.create_subprocess_exec(
|
||||
self.python, "-m", "wsai.backends.whisper_worker",
|
||||
cwd=repo_root, env=env,
|
||||
stdin=asyncio.subprocess.PIPE,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
self._stderr_tail.clear()
|
||||
assert self._proc.stderr is not None
|
||||
self._stderr_task = asyncio.create_task(self._drain_stderr(self._proc.stderr))
|
||||
ready = await self._proc.stdout.readline()
|
||||
if not ready: # worker died before signalling ready
|
||||
await self._proc.wait()
|
||||
raise RuntimeError(
|
||||
f"whisper worker exited before ready (code {self._proc.returncode})."
|
||||
f"{self._stderr_hint()}"
|
||||
async with self._start_lock:
|
||||
# Re-check under the lock: another coroutine may have finished the
|
||||
# (re)start + handshake while we waited.
|
||||
if self._proc is not None and self._proc.returncode is None and self._ready:
|
||||
return
|
||||
self._ready = False
|
||||
env = {
|
||||
**os.environ,
|
||||
"WSAI_WHISPER_MODEL": self.model,
|
||||
"WSAI_WHISPER_DEVICE": self.device,
|
||||
}
|
||||
# ctranslate2 dlopens libcublas/libcudnn from the whisper venv's nvidia
|
||||
# pip packages; the dynamic loader only honours LD_LIBRARY_PATH captured
|
||||
# at exec, so inject those lib dirs into the child env here (harmless on
|
||||
# CPU). Without this the CUDA model loads but transcribe() dies with
|
||||
# "Library libcublas.so.12 is not found".
|
||||
lib_dirs = _cuda_lib_dirs(self.python)
|
||||
if lib_dirs:
|
||||
prev = env.get("LD_LIBRARY_PATH", "")
|
||||
env["LD_LIBRARY_PATH"] = ":".join(lib_dirs + ([prev] if prev else []))
|
||||
if self.language:
|
||||
env["WSAI_WHISPER_LANGUAGE"] = self.language
|
||||
repo_root = str(Path(__file__).resolve().parents[2])
|
||||
self._proc = await asyncio.create_subprocess_exec(
|
||||
self.python, "-m", "wsai.backends.whisper_worker",
|
||||
cwd=repo_root, env=env,
|
||||
stdin=asyncio.subprocess.PIPE,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
try:
|
||||
info = json.loads(ready.decode())
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(
|
||||
f"whisper worker sent invalid ready line {ready!r}: {exc}."
|
||||
f"{self._stderr_hint()}"
|
||||
) from exc
|
||||
if not info.get("ready"):
|
||||
raise RuntimeError(
|
||||
f"whisper worker failed to start: {info}.{self._stderr_hint()}"
|
||||
self._stderr_tail.clear()
|
||||
assert self._proc.stderr is not None
|
||||
self._stderr_task = asyncio.create_task(self._drain_stderr(self._proc.stderr))
|
||||
ready = await self._proc.stdout.readline()
|
||||
if not ready: # worker died before signalling ready
|
||||
await self._proc.wait()
|
||||
raise RuntimeError(
|
||||
f"whisper worker exited before ready (code {self._proc.returncode})."
|
||||
f"{self._stderr_hint()}"
|
||||
)
|
||||
try:
|
||||
info = json.loads(ready.decode())
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(
|
||||
f"whisper worker sent invalid ready line {ready!r}: {exc}."
|
||||
f"{self._stderr_hint()}"
|
||||
) from exc
|
||||
if not info.get("ready"):
|
||||
raise RuntimeError(
|
||||
f"whisper worker failed to start: {info}.{self._stderr_hint()}"
|
||||
)
|
||||
self.load_ms = info.get("ms")
|
||||
self.resolved_device = info.get("device")
|
||||
self._ready = True
|
||||
log.info(
|
||||
"whisper worker ready in %s ms on %s (model %s)",
|
||||
self.load_ms, info.get("device"), info.get("model"),
|
||||
)
|
||||
self.load_ms = info.get("ms")
|
||||
self.resolved_device = info.get("device")
|
||||
log.info(
|
||||
"whisper worker ready in %s ms on %s (model %s)",
|
||||
self.load_ms, info.get("device"), info.get("model"),
|
||||
)
|
||||
|
||||
async def transcribe(self, wav_path: str, *, language: str | None = None) -> str:
|
||||
"""Transcribe one wav file to text using the warm worker."""
|
||||
@@ -211,3 +227,4 @@ class WhisperSTT:
|
||||
pass
|
||||
self._stderr_task = None
|
||||
self._proc = None
|
||||
self._ready = False
|
||||
|
||||
Reference in New Issue
Block a user