fix: serialize worker warmup handshake to stop concurrent stdout reads
The STT/TTS worker _ensure() treated a spawned-but-not-yet-handshaked subprocess as ready, so a voice turn arriving during warmup read the same stdout StreamReader concurrently with the warmup handshake and crashed with "readuntil() called while another coroutine is already waiting for incoming data". Add a _start_lock + _ready flag so (re)start and the ready handshake run atomically and callers wait for real readiness before reading stdout. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -97,6 +97,11 @@ class MeloTTS:
|
||||
self.sink = sink or _log_sink
|
||||
self._proc: asyncio.subprocess.Process | None = None
|
||||
self._lock = asyncio.Lock()
|
||||
# Serialises worker (re)start + the ready handshake so a caller that
|
||||
# arrives mid-warmup waits for readiness instead of reading the same
|
||||
# stdout StreamReader concurrently (asyncio forbids overlapping reads).
|
||||
self._start_lock = asyncio.Lock()
|
||||
self._ready = False # True only after the ready handshake completes
|
||||
self._n = 0
|
||||
self.load_ms: int | None = None
|
||||
# Keep the worker's most recent stderr lines so a crash reports its real
|
||||
@@ -132,42 +137,53 @@ class MeloTTS:
|
||||
await self._ensure()
|
||||
|
||||
async def _ensure(self) -> None:
|
||||
if self._proc is not None and self._proc.returncode is None:
|
||||
# Fast path: only skip when the worker is not just spawned but fully
|
||||
# handshaked. Checking `_proc` alone would let a caller sail past while
|
||||
# another coroutine (e.g. warmup) is still awaiting the ready line on
|
||||
# this same stdout, causing overlapping StreamReader reads.
|
||||
if self._proc is not None and self._proc.returncode is None and self._ready:
|
||||
return
|
||||
self.out_dir.mkdir(parents=True, exist_ok=True)
|
||||
env = {**os.environ, "WSAI_MELO_DEVICE": self.device}
|
||||
# Run the worker module from the wsai source tree with the melo venv.
|
||||
repo_root = str(Path(__file__).resolve().parents[2])
|
||||
self._proc = await asyncio.create_subprocess_exec(
|
||||
self.python, "-m", "wsai.backends.melo_worker",
|
||||
cwd=repo_root, env=env,
|
||||
stdin=asyncio.subprocess.PIPE,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
self._stderr_tail.clear()
|
||||
assert self._proc.stderr is not None
|
||||
self._stderr_task = asyncio.create_task(self._drain_stderr(self._proc.stderr))
|
||||
ready = await self._proc.stdout.readline()
|
||||
if not ready: # worker died before signalling ready
|
||||
await self._proc.wait()
|
||||
raise RuntimeError(
|
||||
f"melo worker exited before ready (code {self._proc.returncode})."
|
||||
f"{self._stderr_hint()}"
|
||||
async with self._start_lock:
|
||||
# Re-check under the lock: another coroutine may have finished the
|
||||
# (re)start + handshake while we waited.
|
||||
if self._proc is not None and self._proc.returncode is None and self._ready:
|
||||
return
|
||||
self._ready = False
|
||||
self.out_dir.mkdir(parents=True, exist_ok=True)
|
||||
env = {**os.environ, "WSAI_MELO_DEVICE": self.device}
|
||||
# Run the worker module from the wsai source tree with the melo venv.
|
||||
repo_root = str(Path(__file__).resolve().parents[2])
|
||||
self._proc = await asyncio.create_subprocess_exec(
|
||||
self.python, "-m", "wsai.backends.melo_worker",
|
||||
cwd=repo_root, env=env,
|
||||
stdin=asyncio.subprocess.PIPE,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
try:
|
||||
info = json.loads(ready.decode())
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(
|
||||
f"melo worker sent invalid ready line {ready!r}: {exc}."
|
||||
f"{self._stderr_hint()}"
|
||||
) from exc
|
||||
if not info.get("ready"):
|
||||
raise RuntimeError(
|
||||
f"melo worker failed to start: {info}.{self._stderr_hint()}"
|
||||
)
|
||||
self.load_ms = info.get("ms")
|
||||
log.info("melo worker ready in %s ms on %s", self.load_ms, info.get("device"))
|
||||
self._stderr_tail.clear()
|
||||
assert self._proc.stderr is not None
|
||||
self._stderr_task = asyncio.create_task(self._drain_stderr(self._proc.stderr))
|
||||
ready = await self._proc.stdout.readline()
|
||||
if not ready: # worker died before signalling ready
|
||||
await self._proc.wait()
|
||||
raise RuntimeError(
|
||||
f"melo worker exited before ready (code {self._proc.returncode})."
|
||||
f"{self._stderr_hint()}"
|
||||
)
|
||||
try:
|
||||
info = json.loads(ready.decode())
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(
|
||||
f"melo worker sent invalid ready line {ready!r}: {exc}."
|
||||
f"{self._stderr_hint()}"
|
||||
) from exc
|
||||
if not info.get("ready"):
|
||||
raise RuntimeError(
|
||||
f"melo worker failed to start: {info}.{self._stderr_hint()}"
|
||||
)
|
||||
self.load_ms = info.get("ms")
|
||||
self._ready = True
|
||||
log.info("melo worker ready in %s ms on %s", self.load_ms, info.get("device"))
|
||||
|
||||
async def synth(self, text: str) -> str:
|
||||
"""Synthesize `text` to a wav and return its path (no sink). Reusable by
|
||||
@@ -223,3 +239,4 @@ class MeloTTS:
|
||||
pass
|
||||
self._stderr_task = None
|
||||
self._proc = None
|
||||
self._ready = False
|
||||
|
||||
Reference in New Issue
Block a user