feat(voice): run pipeline eyes-free (STT -> Brain -> TTS), defer screen share
Make source/vision optional so the conversation loop runs with no screen capture. Add Settings.voice() preset and `python -m wsai --voice` demo. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
6
PLAN.md
6
PLAN.md
@@ -2,6 +2,12 @@
|
|||||||
|
|
||||||
디스코드 화면공유를 실시간으로 함께 보며 **음성으로** 대화하는 AI.
|
디스코드 화면공유를 실시간으로 함께 보며 **음성으로** 대화하는 AI.
|
||||||
|
|
||||||
|
## 방향 전환 (2026-08-11)
|
||||||
|
|
||||||
|
- **화면공유(눈) 부분은 일단 보류**하고, 먼저 **음성 대화 루프 STT → 두뇌 → TTS**를 완성한다.
|
||||||
|
- 파이프라인은 이제 눈 없이(source/vision = None) 돌아간다: `python -m wsai --voice`.
|
||||||
|
- 화면공유 관문 1(DAVE) 성과는 아래에 보존. 나중에 눈을 다시 붙일 때 재사용한다.
|
||||||
|
|
||||||
## 확정된 결정 (2026-08-09)
|
## 확정된 결정 (2026-08-09)
|
||||||
|
|
||||||
1. **캡처 방식 (눈)** — 사용자 요구: 스크린샷/브라우저 캡처 ❌,
|
1. **캡처 방식 (눈)** — 사용자 요구: 스크린샷/브라우저 캡처 ❌,
|
||||||
|
|||||||
@@ -35,6 +35,27 @@ def test_mock_pipeline_runs_and_replies(capsys):
|
|||||||
assert "화면:" in replies[0]
|
assert "화면:" in replies[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_voice_only_pipeline_runs_without_eyes():
|
||||||
|
"""Eyes-free config (no source/vision) still runs STT -> Brain -> TTS."""
|
||||||
|
replies: list[str] = []
|
||||||
|
|
||||||
|
class CapturingTTS(MockTTS):
|
||||||
|
async def speak(self, reply):
|
||||||
|
replies.append(reply.text)
|
||||||
|
|
||||||
|
pipe = Pipeline(
|
||||||
|
brain=MockBrain(),
|
||||||
|
stt=MockSTT(script=["안녕", "잘 있어"], interval=0.05),
|
||||||
|
tts=CapturingTTS(),
|
||||||
|
)
|
||||||
|
|
||||||
|
asyncio.run(asyncio.wait_for(pipe.run(), timeout=5))
|
||||||
|
|
||||||
|
assert len(replies) == 2, "voice loop did not reply to every utterance"
|
||||||
|
# No eyes → the brain must report it has not seen a screen.
|
||||||
|
assert "아직 화면을 못 읽었어요" in replies[0]
|
||||||
|
|
||||||
|
|
||||||
def test_history_is_bounded():
|
def test_history_is_bounded():
|
||||||
pipe = Pipeline(
|
pipe = Pipeline(
|
||||||
source=MockFrameSource(limit=0),
|
source=MockFrameSource(limit=0),
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
"""Entry point.
|
"""Entry point.
|
||||||
|
|
||||||
python -m wsai # mock pipeline (no deps, no keys) — runs a demo
|
python -m wsai # mock pipeline (no deps, no keys) — runs a demo
|
||||||
|
python -m wsai --voice # eyes-free voice loop demo (STT -> Brain -> TTS)
|
||||||
python -m wsai --live # capture this screen + Claude eyes/brain
|
python -m wsai --live # capture this screen + Claude eyes/brain
|
||||||
python -m wsai --env # build from WSAI_* environment variables
|
python -m wsai --env # build from WSAI_* environment variables
|
||||||
|
|
||||||
@@ -25,7 +26,8 @@ async def _run(settings: Settings, demo: bool) -> None:
|
|||||||
from .pipeline import Pipeline
|
from .pipeline import Pipeline
|
||||||
|
|
||||||
pipe = build(settings)
|
pipe = build(settings)
|
||||||
pipe.source = MockFrameSource(interval=0.3, limit=4)
|
if pipe.source is not None: # keep eyes-free configs eyes-free
|
||||||
|
pipe.source = MockFrameSource(interval=0.3, limit=4)
|
||||||
if pipe.stt is not None:
|
if pipe.stt is not None:
|
||||||
pipe.stt = MockSTT(interval=0.4)
|
pipe.stt = MockSTT(interval=0.4)
|
||||||
await pipe.run()
|
await pipe.run()
|
||||||
@@ -35,6 +37,7 @@ async def _run(settings: Settings, demo: bool) -> None:
|
|||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
ap = argparse.ArgumentParser(prog="wsai")
|
ap = argparse.ArgumentParser(prog="wsai")
|
||||||
|
ap.add_argument("--voice", action="store_true", help="eyes-free voice loop (STT -> Brain -> TTS)")
|
||||||
ap.add_argument("--live", action="store_true", help="capture screen + Claude backends")
|
ap.add_argument("--live", action="store_true", help="capture screen + Claude backends")
|
||||||
ap.add_argument("--env", action="store_true", help="build from WSAI_* env vars")
|
ap.add_argument("--env", action="store_true", help="build from WSAI_* env vars")
|
||||||
ap.add_argument("-v", "--verbose", action="store_true")
|
ap.add_argument("-v", "--verbose", action="store_true")
|
||||||
@@ -45,7 +48,9 @@ def main() -> None:
|
|||||||
format="%(levelname)s %(name)s: %(message)s",
|
format="%(levelname)s %(name)s: %(message)s",
|
||||||
)
|
)
|
||||||
|
|
||||||
if args.live:
|
if args.voice:
|
||||||
|
settings, demo = Settings.voice(), True
|
||||||
|
elif args.live:
|
||||||
settings, demo = Settings.live(), False
|
settings, demo = Settings.live(), False
|
||||||
elif args.env:
|
elif args.env:
|
||||||
settings, demo = Settings.from_env(), False
|
settings, demo = Settings.from_env(), False
|
||||||
|
|||||||
@@ -16,12 +16,12 @@ from dataclasses import dataclass
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class Settings:
|
class Settings:
|
||||||
source: str = "mock" # mock | mss
|
source: str | None = "mock" # mock | mss | None (eyes-free)
|
||||||
vision: str = "mock" # mock | claude
|
vision: str | None = "mock" # mock | claude | None (eyes-free)
|
||||||
stt: str | None = "mock" # mock | (whisper) | None
|
stt: str | None = "mock" # mock | (whisper) | None
|
||||||
tts: str | None = "mock" # mock | (melo) | None
|
tts: str | None = "mock" # mock | (melo) | None
|
||||||
brain: str = "mock" # mock | claude
|
brain: str = "mock" # mock | claude
|
||||||
text: str | None = None # None | (discord)
|
text: str | None = None # None | (discord)
|
||||||
|
|
||||||
capture_interval: float = 1.5
|
capture_interval: float = 1.5
|
||||||
anthropic_model: str = "claude-sonnet-4-5"
|
anthropic_model: str = "claude-sonnet-4-5"
|
||||||
@@ -51,3 +51,12 @@ class Settings:
|
|||||||
"""A realistic local config: capture this screen, Claude eyes+brain,
|
"""A realistic local config: capture this screen, Claude eyes+brain,
|
||||||
mock voice (until STT/TTS backends are wired)."""
|
mock voice (until STT/TTS backends are wired)."""
|
||||||
return cls(source="mss", vision="claude", brain="claude", stt="mock", tts="mock")
|
return cls(source="mss", vision="claude", brain="claude", stt="mock", tts="mock")
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def voice(cls) -> "Settings":
|
||||||
|
"""Eyes-free voice loop: no screen share, just STT -> Brain -> TTS.
|
||||||
|
|
||||||
|
Screen capture is deferred, so source/vision are off. Backends default
|
||||||
|
to mock so it runs out of the box; flip stt/tts/brain to real ones as
|
||||||
|
they land."""
|
||||||
|
return cls(source=None, vision=None, stt="mock", tts="mock", brain="mock")
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ def build(settings: Settings) -> Pipeline:
|
|||||||
|
|
||||||
|
|
||||||
def _source(s: Settings):
|
def _source(s: Settings):
|
||||||
|
if s.source in (None, "none"):
|
||||||
|
return None
|
||||||
if s.source == "mss":
|
if s.source == "mss":
|
||||||
from .backends.capture_mss import MSSFrameSource
|
from .backends.capture_mss import MSSFrameSource
|
||||||
|
|
||||||
@@ -29,6 +31,8 @@ def _source(s: Settings):
|
|||||||
|
|
||||||
|
|
||||||
def _vision(s: Settings):
|
def _vision(s: Settings):
|
||||||
|
if s.vision in (None, "none"):
|
||||||
|
return None
|
||||||
if s.vision == "claude":
|
if s.vision == "claude":
|
||||||
from .backends.claude import ClaudeVision
|
from .backends.claude import ClaudeVision
|
||||||
|
|
||||||
|
|||||||
@@ -26,15 +26,16 @@ class Pipeline:
|
|||||||
* perception: FrameSource -> VisionBackend -> SharedScreenContext
|
* perception: FrameSource -> VisionBackend -> SharedScreenContext
|
||||||
* conversation: (SpeechToText | TextChannel) -> Brain -> (TextToSpeech | TextChannel)
|
* conversation: (SpeechToText | TextChannel) -> Brain -> (TextToSpeech | TextChannel)
|
||||||
|
|
||||||
Either input/output half can be None, so you can run text-only, voice-only,
|
Any half can be omitted. With no source/vision it runs eyes-free as a pure
|
||||||
or a headless "just watch" configuration.
|
voice loop (STT -> Brain -> TTS); with no stt/tts it runs text-only; with no
|
||||||
|
conversation it is a headless "just watch" configuration.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
source: FrameSource,
|
source: FrameSource | None = None,
|
||||||
vision: VisionBackend,
|
vision: VisionBackend | None = None,
|
||||||
brain: Brain,
|
brain: Brain,
|
||||||
stt: SpeechToText | None = None,
|
stt: SpeechToText | None = None,
|
||||||
tts: TextToSpeech | None = None,
|
tts: TextToSpeech | None = None,
|
||||||
@@ -53,6 +54,8 @@ class Pipeline:
|
|||||||
|
|
||||||
# -- perception -------------------------------------------------------- #
|
# -- perception -------------------------------------------------------- #
|
||||||
async def _perceive(self) -> None:
|
async def _perceive(self) -> None:
|
||||||
|
if self.source is None or self.vision is None:
|
||||||
|
return # eyes-free (voice-only) configuration
|
||||||
async for frame in self.source.frames():
|
async for frame in self.source.frames():
|
||||||
try:
|
try:
|
||||||
obs = await self.vision.describe(frame)
|
obs = await self.vision.describe(frame)
|
||||||
|
|||||||
Reference in New Issue
Block a user