feat: scaffold watch-screen AI pipeline (mock-runnable skeleton)
Modular async pipeline: FrameSource->Vision->context and STT/text->Brain->TTS. All stages are Protocols; mock backends run end-to-end with no deps/keys. Real backends included: mss screen capture, Claude vision+brain (guarded imports).
This commit is contained in:
136
wsai/interfaces.py
Normal file
136
wsai/interfaces.py
Normal file
@@ -0,0 +1,136 @@
|
||||
"""Core data types and component interfaces for the watch-screen AI.
|
||||
|
||||
The whole system is a small pipeline:
|
||||
|
||||
FrameSource --frames--> VisionBackend --observations--> [SharedScreenContext]
|
||||
|
|
||||
SpeechToText / TextInput --utterances--> Brain <----------------/
|
||||
|
|
||||
v
|
||||
TextToSpeech / TextOutput
|
||||
|
||||
Every stage is a Protocol so a concrete backend (mock, local GPU, cloud API,
|
||||
discord web capture, ...) can be swapped in from config without touching the
|
||||
orchestrator.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import AsyncIterator, Protocol, runtime_checkable
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Data that flows through the pipeline
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
@dataclass
|
||||
class Frame:
|
||||
"""A single captured image of the shared screen."""
|
||||
|
||||
# Raw encoded image bytes (PNG/JPEG). Kept as bytes so any backend can
|
||||
# decode it however it likes and so it is trivial to base64 for a cloud API.
|
||||
data: bytes
|
||||
width: int
|
||||
height: int
|
||||
# Monotonic capture timestamp in seconds.
|
||||
ts: float
|
||||
mime: str = "image/png"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScreenObservation:
|
||||
"""What the vision backend understood from a Frame."""
|
||||
|
||||
text: str
|
||||
ts: float
|
||||
# Optional structured hints (e.g. detected app, code language, error text).
|
||||
tags: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Utterance:
|
||||
"""Something the user said (voice→text) or typed."""
|
||||
|
||||
text: str
|
||||
ts: float
|
||||
source: str = "voice" # "voice" | "text"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Reply:
|
||||
"""The AI's response, ready to be spoken and/or shown."""
|
||||
|
||||
text: str
|
||||
ts: float
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Component interfaces
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
@runtime_checkable
|
||||
class FrameSource(Protocol):
|
||||
"""Produces frames of the shared screen."""
|
||||
|
||||
async def frames(self) -> AsyncIterator[Frame]:
|
||||
"""Yield frames until cancelled. Cadence is up to the implementation."""
|
||||
...
|
||||
|
||||
async def aclose(self) -> None:
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class VisionBackend(Protocol):
|
||||
"""Turns a Frame into a text description of what is on screen."""
|
||||
|
||||
async def describe(self, frame: Frame, hint: str | None = None) -> ScreenObservation:
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SpeechToText(Protocol):
|
||||
"""Streams user utterances from the microphone (or a mock source)."""
|
||||
|
||||
async def utterances(self) -> AsyncIterator[Utterance]:
|
||||
...
|
||||
|
||||
async def aclose(self) -> None:
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class TextToSpeech(Protocol):
|
||||
"""Speaks a reply out loud."""
|
||||
|
||||
async def speak(self, reply: Reply) -> None:
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class Brain(Protocol):
|
||||
"""The conversational LLM. Given the latest screen context, the user's
|
||||
message and the running history, produce a reply."""
|
||||
|
||||
async def respond(
|
||||
self,
|
||||
user_text: str,
|
||||
screen: ScreenObservation | None,
|
||||
history: list[tuple[str, str]],
|
||||
) -> Reply:
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class TextChannel(Protocol):
|
||||
"""Optional text I/O (e.g. a Discord channel) that mirrors the voice loop."""
|
||||
|
||||
async def messages(self) -> AsyncIterator[Utterance]:
|
||||
...
|
||||
|
||||
async def send(self, reply: Reply) -> None:
|
||||
...
|
||||
|
||||
async def aclose(self) -> None:
|
||||
...
|
||||
Reference in New Issue
Block a user