feat(voice): run pipeline eyes-free (STT -> Brain -> TTS), defer screen share

Make source/vision optional so the conversation loop runs with no screen
capture. Add Settings.voice() preset and `python -m wsai --voice` demo.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
EJClaw
2026-08-11 00:05:02 +09:00
parent b0780988a2
commit 2b6059141e
6 changed files with 60 additions and 12 deletions

View File

@@ -1,6 +1,7 @@
"""Entry point.
python -m wsai # mock pipeline (no deps, no keys) — runs a demo
python -m wsai --voice # eyes-free voice loop demo (STT -> Brain -> TTS)
python -m wsai --live # capture this screen + Claude eyes/brain
python -m wsai --env # build from WSAI_* environment variables
@@ -25,7 +26,8 @@ async def _run(settings: Settings, demo: bool) -> None:
from .pipeline import Pipeline
pipe = build(settings)
pipe.source = MockFrameSource(interval=0.3, limit=4)
if pipe.source is not None: # keep eyes-free configs eyes-free
pipe.source = MockFrameSource(interval=0.3, limit=4)
if pipe.stt is not None:
pipe.stt = MockSTT(interval=0.4)
await pipe.run()
@@ -35,6 +37,7 @@ async def _run(settings: Settings, demo: bool) -> None:
def main() -> None:
ap = argparse.ArgumentParser(prog="wsai")
ap.add_argument("--voice", action="store_true", help="eyes-free voice loop (STT -> Brain -> TTS)")
ap.add_argument("--live", action="store_true", help="capture screen + Claude backends")
ap.add_argument("--env", action="store_true", help="build from WSAI_* env vars")
ap.add_argument("-v", "--verbose", action="store_true")
@@ -45,7 +48,9 @@ def main() -> None:
format="%(levelname)s %(name)s: %(message)s",
)
if args.live:
if args.voice:
settings, demo = Settings.voice(), True
elif args.live:
settings, demo = Settings.live(), False
elif args.env:
settings, demo = Settings.from_env(), False

View File

@@ -16,12 +16,12 @@ from dataclasses import dataclass
@dataclass
class Settings:
source: str = "mock" # mock | mss
vision: str = "mock" # mock | claude
stt: str | None = "mock" # mock | (whisper) | None
tts: str | None = "mock" # mock | (melo) | None
brain: str = "mock" # mock | claude
text: str | None = None # None | (discord)
source: str | None = "mock" # mock | mss | None (eyes-free)
vision: str | None = "mock" # mock | claude | None (eyes-free)
stt: str | None = "mock" # mock | (whisper) | None
tts: str | None = "mock" # mock | (melo) | None
brain: str = "mock" # mock | claude
text: str | None = None # None | (discord)
capture_interval: float = 1.5
anthropic_model: str = "claude-sonnet-4-5"
@@ -51,3 +51,12 @@ class Settings:
"""A realistic local config: capture this screen, Claude eyes+brain,
mock voice (until STT/TTS backends are wired)."""
return cls(source="mss", vision="claude", brain="claude", stt="mock", tts="mock")
@classmethod
def voice(cls) -> "Settings":
"""Eyes-free voice loop: no screen share, just STT -> Brain -> TTS.
Screen capture is deferred, so source/vision are off. Backends default
to mock so it runs out of the box; flip stt/tts/brain to real ones as
they land."""
return cls(source=None, vision=None, stt="mock", tts="mock", brain="mock")

View File

@@ -19,6 +19,8 @@ def build(settings: Settings) -> Pipeline:
def _source(s: Settings):
if s.source in (None, "none"):
return None
if s.source == "mss":
from .backends.capture_mss import MSSFrameSource
@@ -29,6 +31,8 @@ def _source(s: Settings):
def _vision(s: Settings):
if s.vision in (None, "none"):
return None
if s.vision == "claude":
from .backends.claude import ClaudeVision

View File

@@ -26,15 +26,16 @@ class Pipeline:
* perception: FrameSource -> VisionBackend -> SharedScreenContext
* conversation: (SpeechToText | TextChannel) -> Brain -> (TextToSpeech | TextChannel)
Either input/output half can be None, so you can run text-only, voice-only,
or a headless "just watch" configuration.
Any half can be omitted. With no source/vision it runs eyes-free as a pure
voice loop (STT -> Brain -> TTS); with no stt/tts it runs text-only; with no
conversation it is a headless "just watch" configuration.
"""
def __init__(
self,
*,
source: FrameSource,
vision: VisionBackend,
source: FrameSource | None = None,
vision: VisionBackend | None = None,
brain: Brain,
stt: SpeechToText | None = None,
tts: TextToSpeech | None = None,
@@ -53,6 +54,8 @@ class Pipeline:
# -- perception -------------------------------------------------------- #
async def _perceive(self) -> None:
if self.source is None or self.vision is None:
return # eyes-free (voice-only) configuration
async for frame in self.source.frames():
try:
obs = await self.vision.describe(frame)