fix: serialize worker warmup handshake to stop concurrent stdout reads

The STT/TTS worker _ensure() treated a spawned-but-not-yet-handshaked
subprocess as ready, so a voice turn arriving during warmup read the same
stdout StreamReader concurrently with the warmup handshake and crashed with
"readuntil() called while another coroutine is already waiting for incoming
data". Add a _start_lock + _ready flag so (re)start and the ready handshake
run atomically and callers wait for real readiness before reading stdout.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
EJClaw
2026-08-22 22:43:43 +09:00
parent 2ea7d04289
commit 6e20f2cd79
3 changed files with 191 additions and 84 deletions

View File

@@ -75,6 +75,11 @@ class WhisperSTT:
self.audio_source = audio_source
self._proc: asyncio.subprocess.Process | None = None
self._lock = asyncio.Lock()
# Serialises worker (re)start + the ready handshake so a caller that
# arrives mid-warmup waits for readiness instead of reading the same
# stdout StreamReader concurrently (asyncio forbids overlapping reads).
self._start_lock = asyncio.Lock()
self._ready = False # True only after the ready handshake completes
self.load_ms: int | None = None
self.resolved_device: str | None = None # "cuda" | "cpu", known after start
# Keep the worker's most recent stderr so a crash reports its real cause
@@ -106,59 +111,70 @@ class WhisperSTT:
await self._ensure()
async def _ensure(self) -> None:
if self._proc is not None and self._proc.returncode is None:
# Fast path: only skip when the worker is not just spawned but fully
# handshaked. Checking `_proc` alone would let a caller sail past while
# another coroutine (e.g. warmup) is still awaiting the ready line on
# this same stdout, causing overlapping StreamReader reads.
if self._proc is not None and self._proc.returncode is None and self._ready:
return
env = {
**os.environ,
"WSAI_WHISPER_MODEL": self.model,
"WSAI_WHISPER_DEVICE": self.device,
}
# ctranslate2 dlopens libcublas/libcudnn from the whisper venv's nvidia
# pip packages; the dynamic loader only honours LD_LIBRARY_PATH captured
# at exec, so inject those lib dirs into the child env here (harmless on
# CPU). Without this the CUDA model loads but transcribe() dies with
# "Library libcublas.so.12 is not found".
lib_dirs = _cuda_lib_dirs(self.python)
if lib_dirs:
prev = env.get("LD_LIBRARY_PATH", "")
env["LD_LIBRARY_PATH"] = ":".join(lib_dirs + ([prev] if prev else []))
if self.language:
env["WSAI_WHISPER_LANGUAGE"] = self.language
repo_root = str(Path(__file__).resolve().parents[2])
self._proc = await asyncio.create_subprocess_exec(
self.python, "-m", "wsai.backends.whisper_worker",
cwd=repo_root, env=env,
stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
self._stderr_tail.clear()
assert self._proc.stderr is not None
self._stderr_task = asyncio.create_task(self._drain_stderr(self._proc.stderr))
ready = await self._proc.stdout.readline()
if not ready: # worker died before signalling ready
await self._proc.wait()
raise RuntimeError(
f"whisper worker exited before ready (code {self._proc.returncode})."
f"{self._stderr_hint()}"
async with self._start_lock:
# Re-check under the lock: another coroutine may have finished the
# (re)start + handshake while we waited.
if self._proc is not None and self._proc.returncode is None and self._ready:
return
self._ready = False
env = {
**os.environ,
"WSAI_WHISPER_MODEL": self.model,
"WSAI_WHISPER_DEVICE": self.device,
}
# ctranslate2 dlopens libcublas/libcudnn from the whisper venv's nvidia
# pip packages; the dynamic loader only honours LD_LIBRARY_PATH captured
# at exec, so inject those lib dirs into the child env here (harmless on
# CPU). Without this the CUDA model loads but transcribe() dies with
# "Library libcublas.so.12 is not found".
lib_dirs = _cuda_lib_dirs(self.python)
if lib_dirs:
prev = env.get("LD_LIBRARY_PATH", "")
env["LD_LIBRARY_PATH"] = ":".join(lib_dirs + ([prev] if prev else []))
if self.language:
env["WSAI_WHISPER_LANGUAGE"] = self.language
repo_root = str(Path(__file__).resolve().parents[2])
self._proc = await asyncio.create_subprocess_exec(
self.python, "-m", "wsai.backends.whisper_worker",
cwd=repo_root, env=env,
stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
try:
info = json.loads(ready.decode())
except json.JSONDecodeError as exc:
raise RuntimeError(
f"whisper worker sent invalid ready line {ready!r}: {exc}."
f"{self._stderr_hint()}"
) from exc
if not info.get("ready"):
raise RuntimeError(
f"whisper worker failed to start: {info}.{self._stderr_hint()}"
self._stderr_tail.clear()
assert self._proc.stderr is not None
self._stderr_task = asyncio.create_task(self._drain_stderr(self._proc.stderr))
ready = await self._proc.stdout.readline()
if not ready: # worker died before signalling ready
await self._proc.wait()
raise RuntimeError(
f"whisper worker exited before ready (code {self._proc.returncode})."
f"{self._stderr_hint()}"
)
try:
info = json.loads(ready.decode())
except json.JSONDecodeError as exc:
raise RuntimeError(
f"whisper worker sent invalid ready line {ready!r}: {exc}."
f"{self._stderr_hint()}"
) from exc
if not info.get("ready"):
raise RuntimeError(
f"whisper worker failed to start: {info}.{self._stderr_hint()}"
)
self.load_ms = info.get("ms")
self.resolved_device = info.get("device")
self._ready = True
log.info(
"whisper worker ready in %s ms on %s (model %s)",
self.load_ms, info.get("device"), info.get("model"),
)
self.load_ms = info.get("ms")
self.resolved_device = info.get("device")
log.info(
"whisper worker ready in %s ms on %s (model %s)",
self.load_ms, info.get("device"), info.get("model"),
)
async def transcribe(self, wav_path: str, *, language: str | None = None) -> str:
"""Transcribe one wav file to text using the warm worker."""
@@ -211,3 +227,4 @@ class WhisperSTT:
pass
self._stderr_task = None
self._proc = None
self._ready = False