tkrmagid/watch_sceen_ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(voice): real Claude brain in the Discord loop (think, not echo)

Browse Source 
The voice loop echoed the recognised text. Wire the real brain: the Discord
voice-turn now runs STT -> ClaudeBrain.respond (with rolling conversation
history) -> TTS, so the bot actually thinks and answers. --voice-server builds
the brain by default (WSAI_BRAIN=claude, WSAI_BRAIN_MODEL overridable) and
gracefully falls back to echo if anthropic/Claude auth is unavailable. A brain
error speaks a short apology instead of killing the loop.

Verified end-to-end: an utterance wav returns X-Heard plus a distinct Claude
X-Reply and a synthesised reply wav on device=cuda. 12 tests pass.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
EJClaw
2026-08-18 23:03:55 +09:00
parent 95e2d4472b
commit 575ac2949a
2 changed files with 50 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
wsai/__main__.py
View File

@@ -113,13 +113,26 @@ def _run_voice_server(host: str, port: int) -> None:
from .dashboard import Dashboard
from .monitor import Monitor
# Real Claude brain (think + reply). If it can't be constructed (no anthropic
# package / no Claude auth), fall back to echo so the loop still works.
brain = None
brain_name = "echo"
if os.environ.get("WSAI_BRAIN", "claude").lower() not in ("none", "echo"):
try:
from .backends.claude import ClaudeBrain
model = os.environ.get("WSAI_BRAIN_MODEL", "claude-sonnet-4-5")
brain = ClaudeBrain(model=model)
brain_name = "claude"
except Exception as exc: # noqa: BLE001
logging.getLogger("wsai").warning("brain disabled (echo fallback): %s", exc)
monitor = Monitor()
stt = WhisperSTT()
tts = MeloTTS()
dash = Dashboard(monitor, host=host, port=port, stt=stt, tts=tts)
dash = Dashboard(monitor, host=host, port=port, stt=stt, tts=tts, brain=brain)
dash.start()
monitor.set_components({"source": "none", "vision": "none", "stt": "whisper",
"brain": "echo", "tts": "melo"})
"brain": brain_name, "tts": "melo"})
monitor.set_status(running=True, listening=False)
monitor.log("info", "디스코드 음성 서버 시작 — STT+TTS GPU 워밍업 중…")
print("\n STT+TTS 워밍업 중… (모델 로드 + CUDA 예열)")
@@ -129,7 +142,7 @@ def _run_voice_server(host: str, port: int) -> None:
monitor.log("info", f"음성 서버 준비 완료 (STT device={sdev}). 디스코드 봇 연결 대기.")
shown = host if host not in ("0.0.0.0", "") else _lan_ip()
print(f"\n 음성 서버 준비 완료 (STT device: {sdev})")
print(f"\n 음성 서버 준비 완료 (STT device: {sdev}, 두뇌: {brain_name})")
print(f" 대시보드/상태: http://{shown}:{port}")
print(f" 봇 연결 엔드포인트: http://127.0.0.1:{port}/api/voice-turn\n")
try:

44
wsai/dashboard.py
View File

@@ -172,12 +172,15 @@ class Dashboard:
"""
def __init__(self, monitor: Monitor, host: str = "0.0.0.0", port: int = 8787,
stt=None, tts=None) -> None:
stt=None, tts=None, brain=None, history_turns: int = 12) -> None:
self.monitor = monitor
self.host = host
self.port = port
self.stt = stt
self.tts = tts
self.brain = brain
self._history: list[tuple[str, str]] = []
self._history_turns = history_turns
self._server: ThreadingHTTPServer | None = None
self._thread: threading.Thread | None = None
self._loop = None
@@ -230,9 +233,9 @@ class Dashboard:
def voice_turn(self, audio_bytes: bytes) -> dict:
"""One Discord voice turn: decode the uploaded utterance, recognise it
on the GPU, produce a reply (echo of what was heard for now), synthesise
it on the GPU, and return {heard, reply, wav} where wav is the reply
audio bytes for the bot to play back into the channel."""
on the GPU, think of a reply (Claude brain if wired, else echo),
synthesise it on the GPU, and return {heard, reply, wav} where wav is the
reply audio bytes for the bot to play back into the channel."""
import os
import subprocess
import time
@@ -253,19 +256,19 @@ class Dashboard:
["ffmpeg", "-y", "-i", src, "-ar", "16000", "-ac", "1", wav],
check=True, capture_output=True,
)
heard = self._submit(self.stt.transcribe(wav)) or ""
heard = (self._submit(self.stt.transcribe(wav)) or "").strip()
turn.heard(heard or "(빈 결과)")
reply_text = heard.strip()
if not reply_text:
# Nothing recognised (silence/noise): skip TTS, tell the bot.
if not heard:
# Nothing recognised (silence/noise): skip the turn, tell the bot.
turn.finish()
return {"heard": heard, "reply": "", "wav": b""}
reply_text = self._think(heard)
turn.replied(reply_text)
out_path = self._submit(self.tts.synth(reply_text))
with open(out_path, "rb") as f:
reply_wav = f.read()
ms = int((time.monotonic() - t0) * 1000)
turn.replied(reply_text)
step = turn.step("STT+TTS(GPU)")
step = turn.step("STT+두뇌+TTS" if self.brain else "STT+TTS(GPU)")
step.ok, step.ms = True, float(ms)
turn._steps.append(step)
turn.finish()
@@ -288,6 +291,27 @@ class Dashboard:
except OSError:
pass
def _think(self, heard: str) -> str:
"""Turn what was heard into a reply. Uses the Claude brain when wired
(with rolling conversation history); falls back to echo if there is no
brain, and to a spoken apology if the brain call fails — so one API hiccup
never kills the voice loop."""
if self.brain is None:
return heard # echo mode
try:
reply = self._submit(self.brain.respond(heard, None, list(self._history)))
text = (reply.text or "").strip()
except Exception as exc: # noqa: BLE001
log.exception("brain failed")
self.monitor.log("error", f"두뇌 응답 실패: {exc}")
return "미안, 지금 잠깐 생각이 안 났어. 다시 말해줄래?"
if not text:
return "음, 뭐라고 해야 할지 모르겠어. 다시 말해줄래?"
self._history.append((heard, text))
if len(self._history) > self._history_turns:
self._history = self._history[-self._history_turns:]
return text
def transcribe_upload(self, audio_bytes: bytes) -> dict:
"""ffmpeg-normalise an uploaded blob to 16 kHz mono wav, transcribe it
on the GPU, and record the result as a monitor turn so it also shows in