4eeddc4b1f
Modular async pipeline: FrameSource->Vision->context and STT/text->Brain->TTS. All stages are Protocols; mock backends run end-to-end with no deps/keys. Real backends included: mss screen capture, Claude vision+brain (guarded imports).
137 lines
3.6 KiB
Python
137 lines
3.6 KiB
Python
"""Core data types and component interfaces for the watch-screen AI.
|
|
|
|
The whole system is a small pipeline:
|
|
|
|
FrameSource --frames--> VisionBackend --observations--> [SharedScreenContext]
|
|
|
|
|
SpeechToText / TextInput --utterances--> Brain <----------------/
|
|
|
|
|
v
|
|
TextToSpeech / TextOutput
|
|
|
|
Every stage is a Protocol so a concrete backend (mock, local GPU, cloud API,
|
|
discord web capture, ...) can be swapped in from config without touching the
|
|
orchestrator.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from typing import AsyncIterator, Protocol, runtime_checkable
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Data that flows through the pipeline
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
@dataclass
|
|
class Frame:
|
|
"""A single captured image of the shared screen."""
|
|
|
|
# Raw encoded image bytes (PNG/JPEG). Kept as bytes so any backend can
|
|
# decode it however it likes and so it is trivial to base64 for a cloud API.
|
|
data: bytes
|
|
width: int
|
|
height: int
|
|
# Monotonic capture timestamp in seconds.
|
|
ts: float
|
|
mime: str = "image/png"
|
|
|
|
|
|
@dataclass
|
|
class ScreenObservation:
|
|
"""What the vision backend understood from a Frame."""
|
|
|
|
text: str
|
|
ts: float
|
|
# Optional structured hints (e.g. detected app, code language, error text).
|
|
tags: dict[str, str] = field(default_factory=dict)
|
|
|
|
|
|
@dataclass
|
|
class Utterance:
|
|
"""Something the user said (voice→text) or typed."""
|
|
|
|
text: str
|
|
ts: float
|
|
source: str = "voice" # "voice" | "text"
|
|
|
|
|
|
@dataclass
|
|
class Reply:
|
|
"""The AI's response, ready to be spoken and/or shown."""
|
|
|
|
text: str
|
|
ts: float
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Component interfaces
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
@runtime_checkable
|
|
class FrameSource(Protocol):
|
|
"""Produces frames of the shared screen."""
|
|
|
|
async def frames(self) -> AsyncIterator[Frame]:
|
|
"""Yield frames until cancelled. Cadence is up to the implementation."""
|
|
...
|
|
|
|
async def aclose(self) -> None:
|
|
...
|
|
|
|
|
|
@runtime_checkable
|
|
class VisionBackend(Protocol):
|
|
"""Turns a Frame into a text description of what is on screen."""
|
|
|
|
async def describe(self, frame: Frame, hint: str | None = None) -> ScreenObservation:
|
|
...
|
|
|
|
|
|
@runtime_checkable
|
|
class SpeechToText(Protocol):
|
|
"""Streams user utterances from the microphone (or a mock source)."""
|
|
|
|
async def utterances(self) -> AsyncIterator[Utterance]:
|
|
...
|
|
|
|
async def aclose(self) -> None:
|
|
...
|
|
|
|
|
|
@runtime_checkable
|
|
class TextToSpeech(Protocol):
|
|
"""Speaks a reply out loud."""
|
|
|
|
async def speak(self, reply: Reply) -> None:
|
|
...
|
|
|
|
|
|
@runtime_checkable
|
|
class Brain(Protocol):
|
|
"""The conversational LLM. Given the latest screen context, the user's
|
|
message and the running history, produce a reply."""
|
|
|
|
async def respond(
|
|
self,
|
|
user_text: str,
|
|
screen: ScreenObservation | None,
|
|
history: list[tuple[str, str]],
|
|
) -> Reply:
|
|
...
|
|
|
|
|
|
@runtime_checkable
|
|
class TextChannel(Protocol):
|
|
"""Optional text I/O (e.g. a Discord channel) that mirrors the voice loop."""
|
|
|
|
async def messages(self) -> AsyncIterator[Utterance]:
|
|
...
|
|
|
|
async def send(self, reply: Reply) -> None:
|
|
...
|
|
|
|
async def aclose(self) -> None:
|
|
...
|