feat(voice): real Claude brain in the Discord loop (think, not echo)

The voice loop echoed the recognised text. Wire the real brain: the Discord
voice-turn now runs STT -> ClaudeBrain.respond (with rolling conversation
history) -> TTS, so the bot actually thinks and answers. --voice-server builds
the brain by default (WSAI_BRAIN=claude, WSAI_BRAIN_MODEL overridable) and
gracefully falls back to echo if anthropic/Claude auth is unavailable. A brain
error speaks a short apology instead of killing the loop.

Verified end-to-end: an utterance wav returns X-Heard plus a distinct Claude
X-Reply and a synthesised reply wav on device=cuda. 12 tests pass.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
EJClaw
2026-08-18 23:03:55 +09:00
parent 95e2d4472b
commit 575ac2949a
2 changed files with 50 additions and 13 deletions

View File

@@ -113,13 +113,26 @@ def _run_voice_server(host: str, port: int) -> None:
from .dashboard import Dashboard from .dashboard import Dashboard
from .monitor import Monitor from .monitor import Monitor
# Real Claude brain (think + reply). If it can't be constructed (no anthropic
# package / no Claude auth), fall back to echo so the loop still works.
brain = None
brain_name = "echo"
if os.environ.get("WSAI_BRAIN", "claude").lower() not in ("none", "echo"):
try:
from .backends.claude import ClaudeBrain
model = os.environ.get("WSAI_BRAIN_MODEL", "claude-sonnet-4-5")
brain = ClaudeBrain(model=model)
brain_name = "claude"
except Exception as exc: # noqa: BLE001
logging.getLogger("wsai").warning("brain disabled (echo fallback): %s", exc)
monitor = Monitor() monitor = Monitor()
stt = WhisperSTT() stt = WhisperSTT()
tts = MeloTTS() tts = MeloTTS()
dash = Dashboard(monitor, host=host, port=port, stt=stt, tts=tts) dash = Dashboard(monitor, host=host, port=port, stt=stt, tts=tts, brain=brain)
dash.start() dash.start()
monitor.set_components({"source": "none", "vision": "none", "stt": "whisper", monitor.set_components({"source": "none", "vision": "none", "stt": "whisper",
"brain": "echo", "tts": "melo"}) "brain": brain_name, "tts": "melo"})
monitor.set_status(running=True, listening=False) monitor.set_status(running=True, listening=False)
monitor.log("info", "디스코드 음성 서버 시작 — STT+TTS GPU 워밍업 중…") monitor.log("info", "디스코드 음성 서버 시작 — STT+TTS GPU 워밍업 중…")
print("\n STT+TTS 워밍업 중… (모델 로드 + CUDA 예열)") print("\n STT+TTS 워밍업 중… (모델 로드 + CUDA 예열)")
@@ -129,7 +142,7 @@ def _run_voice_server(host: str, port: int) -> None:
monitor.log("info", f"음성 서버 준비 완료 (STT device={sdev}). 디스코드 봇 연결 대기.") monitor.log("info", f"음성 서버 준비 완료 (STT device={sdev}). 디스코드 봇 연결 대기.")
shown = host if host not in ("0.0.0.0", "") else _lan_ip() shown = host if host not in ("0.0.0.0", "") else _lan_ip()
print(f"\n 음성 서버 준비 완료 (STT device: {sdev})") print(f"\n 음성 서버 준비 완료 (STT device: {sdev}, 두뇌: {brain_name})")
print(f" 대시보드/상태: http://{shown}:{port}") print(f" 대시보드/상태: http://{shown}:{port}")
print(f" 봇 연결 엔드포인트: http://127.0.0.1:{port}/api/voice-turn\n") print(f" 봇 연결 엔드포인트: http://127.0.0.1:{port}/api/voice-turn\n")
try: try:

View File

@@ -172,12 +172,15 @@ class Dashboard:
""" """
def __init__(self, monitor: Monitor, host: str = "0.0.0.0", port: int = 8787, def __init__(self, monitor: Monitor, host: str = "0.0.0.0", port: int = 8787,
stt=None, tts=None) -> None: stt=None, tts=None, brain=None, history_turns: int = 12) -> None:
self.monitor = monitor self.monitor = monitor
self.host = host self.host = host
self.port = port self.port = port
self.stt = stt self.stt = stt
self.tts = tts self.tts = tts
self.brain = brain
self._history: list[tuple[str, str]] = []
self._history_turns = history_turns
self._server: ThreadingHTTPServer | None = None self._server: ThreadingHTTPServer | None = None
self._thread: threading.Thread | None = None self._thread: threading.Thread | None = None
self._loop = None self._loop = None
@@ -230,9 +233,9 @@ class Dashboard:
def voice_turn(self, audio_bytes: bytes) -> dict: def voice_turn(self, audio_bytes: bytes) -> dict:
"""One Discord voice turn: decode the uploaded utterance, recognise it """One Discord voice turn: decode the uploaded utterance, recognise it
on the GPU, produce a reply (echo of what was heard for now), synthesise on the GPU, think of a reply (Claude brain if wired, else echo),
it on the GPU, and return {heard, reply, wav} where wav is the reply synthesise it on the GPU, and return {heard, reply, wav} where wav is the
audio bytes for the bot to play back into the channel.""" reply audio bytes for the bot to play back into the channel."""
import os import os
import subprocess import subprocess
import time import time
@@ -253,19 +256,19 @@ class Dashboard:
["ffmpeg", "-y", "-i", src, "-ar", "16000", "-ac", "1", wav], ["ffmpeg", "-y", "-i", src, "-ar", "16000", "-ac", "1", wav],
check=True, capture_output=True, check=True, capture_output=True,
) )
heard = self._submit(self.stt.transcribe(wav)) or "" heard = (self._submit(self.stt.transcribe(wav)) or "").strip()
turn.heard(heard or "(빈 결과)") turn.heard(heard or "(빈 결과)")
reply_text = heard.strip() if not heard:
if not reply_text: # Nothing recognised (silence/noise): skip the turn, tell the bot.
# Nothing recognised (silence/noise): skip TTS, tell the bot.
turn.finish() turn.finish()
return {"heard": heard, "reply": "", "wav": b""} return {"heard": heard, "reply": "", "wav": b""}
reply_text = self._think(heard)
turn.replied(reply_text)
out_path = self._submit(self.tts.synth(reply_text)) out_path = self._submit(self.tts.synth(reply_text))
with open(out_path, "rb") as f: with open(out_path, "rb") as f:
reply_wav = f.read() reply_wav = f.read()
ms = int((time.monotonic() - t0) * 1000) ms = int((time.monotonic() - t0) * 1000)
turn.replied(reply_text) step = turn.step("STT+두뇌+TTS" if self.brain else "STT+TTS(GPU)")
step = turn.step("STT+TTS(GPU)")
step.ok, step.ms = True, float(ms) step.ok, step.ms = True, float(ms)
turn._steps.append(step) turn._steps.append(step)
turn.finish() turn.finish()
@@ -288,6 +291,27 @@ class Dashboard:
except OSError: except OSError:
pass pass
def _think(self, heard: str) -> str:
"""Turn what was heard into a reply. Uses the Claude brain when wired
(with rolling conversation history); falls back to echo if there is no
brain, and to a spoken apology if the brain call fails — so one API hiccup
never kills the voice loop."""
if self.brain is None:
return heard # echo mode
try:
reply = self._submit(self.brain.respond(heard, None, list(self._history)))
text = (reply.text or "").strip()
except Exception as exc: # noqa: BLE001
log.exception("brain failed")
self.monitor.log("error", f"두뇌 응답 실패: {exc}")
return "미안, 지금 잠깐 생각이 안 났어. 다시 말해줄래?"
if not text:
return "음, 뭐라고 해야 할지 모르겠어. 다시 말해줄래?"
self._history.append((heard, text))
if len(self._history) > self._history_turns:
self._history = self._history[-self._history_turns:]
return text
def transcribe_upload(self, audio_bytes: bytes) -> dict: def transcribe_upload(self, audio_bytes: bytes) -> dict:
"""ffmpeg-normalise an uploaded blob to 16 kHz mono wav, transcribe it """ffmpeg-normalise an uploaded blob to 16 kHz mono wav, transcribe it
on the GPU, and record the result as a monitor turn so it also shows in on the GPU, and record the result as a monitor turn so it also shows in