"""Core data types and component interfaces for the watch-screen AI. The whole system is a small pipeline: FrameSource --frames--> VisionBackend --observations--> [SharedScreenContext] | SpeechToText / TextInput --utterances--> Brain <----------------/ | v TextToSpeech / TextOutput Every stage is a Protocol so a concrete backend (mock, local GPU, cloud API, discord web capture, ...) can be swapped in from config without touching the orchestrator. """ from __future__ import annotations from dataclasses import dataclass, field from typing import AsyncIterator, Protocol, runtime_checkable # --------------------------------------------------------------------------- # # Data that flows through the pipeline # --------------------------------------------------------------------------- # @dataclass class Frame: """A single captured image of the shared screen.""" # Raw encoded image bytes (PNG/JPEG). Kept as bytes so any backend can # decode it however it likes and so it is trivial to base64 for a cloud API. data: bytes width: int height: int # Monotonic capture timestamp in seconds. ts: float mime: str = "image/png" @dataclass class ScreenObservation: """What the vision backend understood from a Frame.""" text: str ts: float # Optional structured hints (e.g. detected app, code language, error text). tags: dict[str, str] = field(default_factory=dict) @dataclass class Utterance: """Something the user said (voice→text) or typed.""" text: str ts: float source: str = "voice" # "voice" | "text" @dataclass class Reply: """The AI's response, ready to be spoken and/or shown.""" text: str ts: float # --------------------------------------------------------------------------- # # Component interfaces # --------------------------------------------------------------------------- # @runtime_checkable class FrameSource(Protocol): """Produces frames of the shared screen.""" async def frames(self) -> AsyncIterator[Frame]: """Yield frames until cancelled. Cadence is up to the implementation.""" ... async def aclose(self) -> None: ... @runtime_checkable class VisionBackend(Protocol): """Turns a Frame into a text description of what is on screen.""" async def describe(self, frame: Frame, hint: str | None = None) -> ScreenObservation: ... @runtime_checkable class SpeechToText(Protocol): """Streams user utterances from the microphone (or a mock source).""" async def utterances(self) -> AsyncIterator[Utterance]: ... async def aclose(self) -> None: ... @runtime_checkable class TextToSpeech(Protocol): """Speaks a reply out loud.""" async def speak(self, reply: Reply) -> None: ... @runtime_checkable class Brain(Protocol): """The conversational LLM. Given the latest screen context, the user's message and the running history, produce a reply.""" async def respond( self, user_text: str, screen: ScreenObservation | None, history: list[tuple[str, str]], ) -> Reply: ... @runtime_checkable class TextChannel(Protocol): """Optional text I/O (e.g. a Discord channel) that mirrors the voice loop.""" async def messages(self) -> AsyncIterator[Utterance]: ... async def send(self, reply: Reply) -> None: ... async def aclose(self) -> None: ...