feat: scaffold watch-screen AI pipeline (mock-runnable skeleton)
Modular async pipeline: FrameSource->Vision->context and STT/text->Brain->TTS. All stages are Protocols; mock backends run end-to-end with no deps/keys. Real backends included: mss screen capture, Claude vision+brain (guarded imports).
This commit is contained in:
67
wsai/backends/capture_mss.py
Normal file
67
wsai/backends/capture_mss.py
Normal file
@@ -0,0 +1,67 @@
|
||||
"""Local screen capture via `mss`.
|
||||
|
||||
This is the practical "eye": run this on the machine that is in the Discord call
|
||||
viewing the shared screen, and it captures that monitor/region. Swap in a
|
||||
discord-web capture later without touching the pipeline.
|
||||
|
||||
Requires: pip install mss pillow
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import time
|
||||
from typing import AsyncIterator
|
||||
|
||||
from ..interfaces import Frame
|
||||
|
||||
|
||||
class MSSFrameSource:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
monitor: int = 1,
|
||||
region: dict | None = None,
|
||||
interval: float = 1.5,
|
||||
max_width: int = 1280,
|
||||
) -> None:
|
||||
# `region` overrides `monitor`: {"top":.., "left":.., "width":.., "height":..}
|
||||
self.monitor = monitor
|
||||
self.region = region
|
||||
self.interval = interval
|
||||
self.max_width = max_width
|
||||
self._sct = None
|
||||
|
||||
def _ensure(self):
|
||||
if self._sct is None:
|
||||
import mss # lazy import so mock mode needs no dependency
|
||||
|
||||
self._sct = mss.mss()
|
||||
return self._sct
|
||||
|
||||
def _grab_png(self) -> tuple[bytes, int, int]:
|
||||
from PIL import Image
|
||||
|
||||
sct = self._ensure()
|
||||
area = self.region or sct.monitors[self.monitor]
|
||||
shot = sct.grab(area)
|
||||
img = Image.frombytes("RGB", shot.size, shot.rgb)
|
||||
if img.width > self.max_width:
|
||||
h = int(img.height * self.max_width / img.width)
|
||||
img = img.resize((self.max_width, h))
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="PNG")
|
||||
return buf.getvalue(), img.width, img.height
|
||||
|
||||
async def frames(self) -> AsyncIterator[Frame]:
|
||||
while True:
|
||||
# mss is blocking; keep the event loop free.
|
||||
data, w, h = await asyncio.to_thread(self._grab_png)
|
||||
yield Frame(data=data, width=w, height=h, ts=time.monotonic(), mime="image/png")
|
||||
await asyncio.sleep(self.interval)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
if self._sct is not None:
|
||||
self._sct.close()
|
||||
self._sct = None
|
||||
Reference in New Issue
Block a user