feat: scaffold watch-screen AI pipeline (mock-runnable skeleton)

Modular async pipeline: FrameSource->Vision->context and STT/text->Brain->TTS.
All stages are Protocols; mock backends run end-to-end with no deps/keys.
Real backends included: mss screen capture, Claude vision+brain (guarded imports).

wsai/backends/capture_mss.py

@@ -0,0 +1,67 @@
+"""Local screen capture via `mss`.
+
+This is the practical "eye": run this on the machine that is in the Discord call
+viewing the shared screen, and it captures that monitor/region. Swap in a
+discord-web capture later without touching the pipeline.
+
+Requires: pip install mss pillow
+"""
+
+from __future__ import annotations
+
+import asyncio
+import io
+import time
+from typing import AsyncIterator
+
+from ..interfaces import Frame
+
+
+class MSSFrameSource:
+    def __init__(
+        self,
+        *,
+        monitor: int = 1,
+        region: dict | None = None,
+        interval: float = 1.5,
+        max_width: int = 1280,
+    ) -> None:
+        # `region` overrides `monitor`: {"top":.., "left":.., "width":.., "height":..}
+        self.monitor = monitor
+        self.region = region
+        self.interval = interval
+        self.max_width = max_width
+        self._sct = None
+
+    def _ensure(self):
+        if self._sct is None:
+            import mss  # lazy import so mock mode needs no dependency
+
+            self._sct = mss.mss()
+        return self._sct
+
+    def _grab_png(self) -> tuple[bytes, int, int]:
+        from PIL import Image
+
+        sct = self._ensure()
+        area = self.region or sct.monitors[self.monitor]
+        shot = sct.grab(area)
+        img = Image.frombytes("RGB", shot.size, shot.rgb)
+        if img.width > self.max_width:
+            h = int(img.height * self.max_width / img.width)
+            img = img.resize((self.max_width, h))
+        buf = io.BytesIO()
+        img.save(buf, format="PNG")
+        return buf.getvalue(), img.width, img.height
+
+    async def frames(self) -> AsyncIterator[Frame]:
+        while True:
+            # mss is blocking; keep the event loop free.
+            data, w, h = await asyncio.to_thread(self._grab_png)
+            yield Frame(data=data, width=w, height=h, ts=time.monotonic(), mime="image/png")
+            await asyncio.sleep(self.interval)
+
+    async def aclose(self) -> None:
+        if self._sct is not None:
+            self._sct.close()
+            self._sct = None

wsai/backends/claude.py

@@ -0,0 +1,81 @@
+"""Cloud brain + vision via the Anthropic (Claude) API.
+
+Both share one client. Vision sends the frame as a base64 image; the brain is a
+plain chat call that receives the latest screen description as context.
+
+Requires: pip install anthropic
+Env: ANTHROPIC_API_KEY
+"""
+
+from __future__ import annotations
+
+import base64
+import os
+import time
+
+from ..interfaces import Frame, Reply, ScreenObservation
+
+
+def _client(api_key: str | None):
+    import anthropic
+
+    return anthropic.AsyncAnthropic(api_key=api_key or os.environ.get("ANTHROPIC_API_KEY"))
+
+
+class ClaudeVision:
+    def __init__(self, *, model: str = "claude-sonnet-4-5", api_key: str | None = None) -> None:
+        self.model = model
+        self._client = _client(api_key)
+
+    async def describe(self, frame: Frame, hint: str | None = None) -> ScreenObservation:
+        prompt = hint or (
+            "이건 디스코드 화면공유 캡처야. 지금 화면에서 무슨 일이 벌어지는지 "
+            "2~3문장으로 한국어로 간결하게 설명해줘. 코드/에러/게임/문서 등 맥락을 짚어줘."
+        )
+        b64 = base64.b64encode(frame.data).decode()
+        resp = await self._client.messages.create(
+            model=self.model,
+            max_tokens=300,
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "image",
+                            "source": {"type": "base64", "media_type": frame.mime, "data": b64},
+                        },
+                        {"type": "text", "text": prompt},
+                    ],
+                }
+            ],
+        )
+        text = "".join(b.text for b in resp.content if b.type == "text")
+        return ScreenObservation(text=text.strip(), ts=frame.ts)
+
+
+class ClaudeBrain:
+    SYSTEM = (
+        "너는 사용자의 디스코드 화면공유를 실시간으로 함께 보는 AI 파트너야. "
+        "화면 설명을 참고해 자연스러운 반말/존댓말은 사용자에 맞추고, 짧고 대화하듯 답해. "
+        "화면을 못 봤으면 솔직히 말해."
+    )
+
+    def __init__(self, *, model: str = "claude-sonnet-4-5", api_key: str | None = None) -> None:
+        self.model = model
+        self._client = _client(api_key)
+
+    async def respond(self, user_text, screen, history) -> Reply:
+        msgs = []
+        for user, ai in history:
+            msgs.append({"role": "user", "content": user})
+            msgs.append({"role": "assistant", "content": ai})
+        screen_note = f"[지금 화면] {screen.text}\n\n" if screen else "[지금 화면] (아직 못 읽음)\n\n"
+        msgs.append({"role": "user", "content": screen_note + user_text})
+        resp = await self._client.messages.create(
+            model=self.model,
+            max_tokens=400,
+            system=self.SYSTEM,
+            messages=msgs,
+        )
+        text = "".join(b.text for b in resp.content if b.type == "text")
+        return Reply(text=text.strip(), ts=time.monotonic())

wsai/backends/mock.py

@@ -0,0 +1,99 @@
+"""Mock backends. These let the full pipeline run with no GPU, no mic, no API
+key — so the skeleton is verifiable and gives every real backend a reference
+implementation to match.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import itertools
+import time
+from typing import AsyncIterator
+
+from ..interfaces import (
+    Frame,
+    Reply,
+    ScreenObservation,
+    Utterance,
+)
+
+
+class MockFrameSource:
+    """Emits tiny synthetic frames on a fixed interval."""
+
+    def __init__(self, interval: float = 1.0, limit: int | None = None) -> None:
+        self.interval = interval
+        self.limit = limit
+
+    async def frames(self) -> AsyncIterator[Frame]:
+        for i in itertools.count():
+            if self.limit is not None and i >= self.limit:
+                return
+            yield Frame(
+                data=b"\x89PNG\r\n\x1a\n",  # PNG magic; enough for a stub
+                width=1280,
+                height=720,
+                ts=time.monotonic(),
+                mime="image/png",
+            )
+            await asyncio.sleep(self.interval)
+
+    async def aclose(self) -> None:  # nothing to release
+        return
+
+
+class MockVision:
+    """Pretends to read the screen. Cycles through a few canned scenes."""
+
+    SCENES = [
+        "VS Code is open with a Python file; a traceback is visible in the terminal.",
+        "A browser shows a GitHub pull request diff.",
+        "A game is running; the player is in a menu screen.",
+    ]
+
+    def __init__(self) -> None:
+        self._i = 0
+
+    async def describe(self, frame: Frame, hint: str | None = None) -> ScreenObservation:
+        scene = self.SCENES[self._i % len(self.SCENES)]
+        self._i += 1
+        return ScreenObservation(text=scene, ts=frame.ts)
+
+
+class MockSTT:
+    """Feeds a scripted set of user utterances, then goes quiet."""
+
+    def __init__(self, script: list[str] | None = None, interval: float = 2.0) -> None:
+        self.script = script or [
+            "지금 화면에 뭐 보여?",
+            "저 에러 왜 나는 거야?",
+            "고마워",
+        ]
+        self.interval = interval
+
+    async def utterances(self) -> AsyncIterator[Utterance]:
+        for line in self.script:
+            await asyncio.sleep(self.interval)
+            yield Utterance(text=line, ts=time.monotonic(), source="voice")
+
+    async def aclose(self) -> None:
+        return
+
+
+class MockTTS:
+    """'Speaks' by printing. Real TTS swaps in here."""
+
+    async def speak(self, reply: Reply) -> None:
+        print(f"[TTS] {reply.text}")
+
+
+class MockBrain:
+    """Echo-style brain that references the current screen, so you can see the
+    screen context actually reaching the conversation loop."""
+
+    async def respond(self, user_text, screen, history) -> Reply:
+        seen = screen.text if screen else "아직 화면을 못 읽었어요"
+        return Reply(
+            text=f'(화면: "{seen}") 라고 봤어요. 말씀하신 "{user_text}"에 대해 답하자면… [mock]',
+            ts=time.monotonic(),
+        )