"""Orchestrator: wires the perception loop and the conversation loop together.""" from __future__ import annotations import asyncio import logging from .interfaces import ( Brain, FrameSource, Reply, SpeechToText, TextChannel, TextToSpeech, Utterance, VisionBackend, ) from .state import SharedScreenContext log = logging.getLogger("wsai.pipeline") class Pipeline: """Runs two concurrent loops: * perception: FrameSource -> VisionBackend -> SharedScreenContext * conversation: (SpeechToText | TextChannel) -> Brain -> (TextToSpeech | TextChannel) Any half can be omitted. With no source/vision it runs eyes-free as a pure voice loop (STT -> Brain -> TTS); with no stt/tts it runs text-only; with no conversation it is a headless "just watch" configuration. """ def __init__( self, *, source: FrameSource | None = None, vision: VisionBackend | None = None, brain: Brain, stt: SpeechToText | None = None, tts: TextToSpeech | None = None, text_channel: TextChannel | None = None, history_turns: int = 12, ) -> None: self.source = source self.vision = vision self.brain = brain self.stt = stt self.tts = tts self.text_channel = text_channel self.context = SharedScreenContext() self._history: list[tuple[str, str]] = [] self._history_turns = history_turns # -- perception -------------------------------------------------------- # async def _perceive(self) -> None: if self.source is None or self.vision is None: return # eyes-free (voice-only) configuration async for frame in self.source.frames(): try: obs = await self.vision.describe(frame) except Exception: # a single bad frame must not kill the loop log.exception("vision.describe failed") continue await self.context.update(obs) log.debug("screen: %s", obs.text[:120]) # -- conversation ------------------------------------------------------ # async def _handle(self, utt: Utterance) -> None: screen = await self.context.latest() reply = await self.brain.respond(utt.text, screen, self._history) self._remember(utt.text, reply.text) await self._emit(reply) def _remember(self, user: str, ai: str) -> None: self._history.append((user, ai)) if len(self._history) > self._history_turns: self._history = self._history[-self._history_turns :] async def _emit(self, reply: Reply) -> None: tasks = [] if self.tts is not None: tasks.append(self.tts.speak(reply)) if self.text_channel is not None: tasks.append(self.text_channel.send(reply)) if not tasks: log.info("AI: %s", reply.text) else: await asyncio.gather(*tasks) async def _listen_voice(self) -> None: if self.stt is None: return async for utt in self.stt.utterances(): await self._handle(utt) async def _listen_text(self) -> None: if self.text_channel is None: return async for utt in self.text_channel.messages(): await self._handle(utt) # -- lifecycle --------------------------------------------------------- # async def run(self) -> None: # A TaskGroup (not bare gather) so that if ONE loop raises, the others # are cancelled and awaited before teardown. With plain gather the # failing loop propagated while the siblings kept running detached, and # aclose() in the finally then closed a source/stt out from under a # still-live loop (close-during-use). try: async with asyncio.TaskGroup() as tg: tg.create_task(self._perceive()) tg.create_task(self._listen_voice()) tg.create_task(self._listen_text()) finally: await self.aclose() async def aclose(self) -> None: for closer in (self.source, self.stt, self.text_channel): if closer is not None: try: await closer.aclose() except Exception: log.exception("error closing %s", closer)