feat(voice): real Claude brain in the Discord loop (think, not echo)
The voice loop echoed the recognised text. Wire the real brain: the Discord voice-turn now runs STT -> ClaudeBrain.respond (with rolling conversation history) -> TTS, so the bot actually thinks and answers. --voice-server builds the brain by default (WSAI_BRAIN=claude, WSAI_BRAIN_MODEL overridable) and gracefully falls back to echo if anthropic/Claude auth is unavailable. A brain error speaks a short apology instead of killing the loop. Verified end-to-end: an utterance wav returns X-Heard plus a distinct Claude X-Reply and a synthesised reply wav on device=cuda. 12 tests pass. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -113,13 +113,26 @@ def _run_voice_server(host: str, port: int) -> None:
|
|||||||
from .dashboard import Dashboard
|
from .dashboard import Dashboard
|
||||||
from .monitor import Monitor
|
from .monitor import Monitor
|
||||||
|
|
||||||
|
# Real Claude brain (think + reply). If it can't be constructed (no anthropic
|
||||||
|
# package / no Claude auth), fall back to echo so the loop still works.
|
||||||
|
brain = None
|
||||||
|
brain_name = "echo"
|
||||||
|
if os.environ.get("WSAI_BRAIN", "claude").lower() not in ("none", "echo"):
|
||||||
|
try:
|
||||||
|
from .backends.claude import ClaudeBrain
|
||||||
|
model = os.environ.get("WSAI_BRAIN_MODEL", "claude-sonnet-4-5")
|
||||||
|
brain = ClaudeBrain(model=model)
|
||||||
|
brain_name = "claude"
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
logging.getLogger("wsai").warning("brain disabled (echo fallback): %s", exc)
|
||||||
|
|
||||||
monitor = Monitor()
|
monitor = Monitor()
|
||||||
stt = WhisperSTT()
|
stt = WhisperSTT()
|
||||||
tts = MeloTTS()
|
tts = MeloTTS()
|
||||||
dash = Dashboard(monitor, host=host, port=port, stt=stt, tts=tts)
|
dash = Dashboard(monitor, host=host, port=port, stt=stt, tts=tts, brain=brain)
|
||||||
dash.start()
|
dash.start()
|
||||||
monitor.set_components({"source": "none", "vision": "none", "stt": "whisper",
|
monitor.set_components({"source": "none", "vision": "none", "stt": "whisper",
|
||||||
"brain": "echo", "tts": "melo"})
|
"brain": brain_name, "tts": "melo"})
|
||||||
monitor.set_status(running=True, listening=False)
|
monitor.set_status(running=True, listening=False)
|
||||||
monitor.log("info", "디스코드 음성 서버 시작 — STT+TTS GPU 워밍업 중…")
|
monitor.log("info", "디스코드 음성 서버 시작 — STT+TTS GPU 워밍업 중…")
|
||||||
print("\n STT+TTS 워밍업 중… (모델 로드 + CUDA 예열)")
|
print("\n STT+TTS 워밍업 중… (모델 로드 + CUDA 예열)")
|
||||||
@@ -129,7 +142,7 @@ def _run_voice_server(host: str, port: int) -> None:
|
|||||||
monitor.log("info", f"음성 서버 준비 완료 (STT device={sdev}). 디스코드 봇 연결 대기.")
|
monitor.log("info", f"음성 서버 준비 완료 (STT device={sdev}). 디스코드 봇 연결 대기.")
|
||||||
|
|
||||||
shown = host if host not in ("0.0.0.0", "") else _lan_ip()
|
shown = host if host not in ("0.0.0.0", "") else _lan_ip()
|
||||||
print(f"\n 음성 서버 준비 완료 (STT device: {sdev})")
|
print(f"\n 음성 서버 준비 완료 (STT device: {sdev}, 두뇌: {brain_name})")
|
||||||
print(f" 대시보드/상태: http://{shown}:{port}")
|
print(f" 대시보드/상태: http://{shown}:{port}")
|
||||||
print(f" 봇 연결 엔드포인트: http://127.0.0.1:{port}/api/voice-turn\n")
|
print(f" 봇 연결 엔드포인트: http://127.0.0.1:{port}/api/voice-turn\n")
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -172,12 +172,15 @@ class Dashboard:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, monitor: Monitor, host: str = "0.0.0.0", port: int = 8787,
|
def __init__(self, monitor: Monitor, host: str = "0.0.0.0", port: int = 8787,
|
||||||
stt=None, tts=None) -> None:
|
stt=None, tts=None, brain=None, history_turns: int = 12) -> None:
|
||||||
self.monitor = monitor
|
self.monitor = monitor
|
||||||
self.host = host
|
self.host = host
|
||||||
self.port = port
|
self.port = port
|
||||||
self.stt = stt
|
self.stt = stt
|
||||||
self.tts = tts
|
self.tts = tts
|
||||||
|
self.brain = brain
|
||||||
|
self._history: list[tuple[str, str]] = []
|
||||||
|
self._history_turns = history_turns
|
||||||
self._server: ThreadingHTTPServer | None = None
|
self._server: ThreadingHTTPServer | None = None
|
||||||
self._thread: threading.Thread | None = None
|
self._thread: threading.Thread | None = None
|
||||||
self._loop = None
|
self._loop = None
|
||||||
@@ -230,9 +233,9 @@ class Dashboard:
|
|||||||
|
|
||||||
def voice_turn(self, audio_bytes: bytes) -> dict:
|
def voice_turn(self, audio_bytes: bytes) -> dict:
|
||||||
"""One Discord voice turn: decode the uploaded utterance, recognise it
|
"""One Discord voice turn: decode the uploaded utterance, recognise it
|
||||||
on the GPU, produce a reply (echo of what was heard for now), synthesise
|
on the GPU, think of a reply (Claude brain if wired, else echo),
|
||||||
it on the GPU, and return {heard, reply, wav} where wav is the reply
|
synthesise it on the GPU, and return {heard, reply, wav} where wav is the
|
||||||
audio bytes for the bot to play back into the channel."""
|
reply audio bytes for the bot to play back into the channel."""
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
import time
|
import time
|
||||||
@@ -253,19 +256,19 @@ class Dashboard:
|
|||||||
["ffmpeg", "-y", "-i", src, "-ar", "16000", "-ac", "1", wav],
|
["ffmpeg", "-y", "-i", src, "-ar", "16000", "-ac", "1", wav],
|
||||||
check=True, capture_output=True,
|
check=True, capture_output=True,
|
||||||
)
|
)
|
||||||
heard = self._submit(self.stt.transcribe(wav)) or ""
|
heard = (self._submit(self.stt.transcribe(wav)) or "").strip()
|
||||||
turn.heard(heard or "(빈 결과)")
|
turn.heard(heard or "(빈 결과)")
|
||||||
reply_text = heard.strip()
|
if not heard:
|
||||||
if not reply_text:
|
# Nothing recognised (silence/noise): skip the turn, tell the bot.
|
||||||
# Nothing recognised (silence/noise): skip TTS, tell the bot.
|
|
||||||
turn.finish()
|
turn.finish()
|
||||||
return {"heard": heard, "reply": "", "wav": b""}
|
return {"heard": heard, "reply": "", "wav": b""}
|
||||||
|
reply_text = self._think(heard)
|
||||||
|
turn.replied(reply_text)
|
||||||
out_path = self._submit(self.tts.synth(reply_text))
|
out_path = self._submit(self.tts.synth(reply_text))
|
||||||
with open(out_path, "rb") as f:
|
with open(out_path, "rb") as f:
|
||||||
reply_wav = f.read()
|
reply_wav = f.read()
|
||||||
ms = int((time.monotonic() - t0) * 1000)
|
ms = int((time.monotonic() - t0) * 1000)
|
||||||
turn.replied(reply_text)
|
step = turn.step("STT+두뇌+TTS" if self.brain else "STT+TTS(GPU)")
|
||||||
step = turn.step("STT+TTS(GPU)")
|
|
||||||
step.ok, step.ms = True, float(ms)
|
step.ok, step.ms = True, float(ms)
|
||||||
turn._steps.append(step)
|
turn._steps.append(step)
|
||||||
turn.finish()
|
turn.finish()
|
||||||
@@ -288,6 +291,27 @@ class Dashboard:
|
|||||||
except OSError:
|
except OSError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
def _think(self, heard: str) -> str:
|
||||||
|
"""Turn what was heard into a reply. Uses the Claude brain when wired
|
||||||
|
(with rolling conversation history); falls back to echo if there is no
|
||||||
|
brain, and to a spoken apology if the brain call fails — so one API hiccup
|
||||||
|
never kills the voice loop."""
|
||||||
|
if self.brain is None:
|
||||||
|
return heard # echo mode
|
||||||
|
try:
|
||||||
|
reply = self._submit(self.brain.respond(heard, None, list(self._history)))
|
||||||
|
text = (reply.text or "").strip()
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
log.exception("brain failed")
|
||||||
|
self.monitor.log("error", f"두뇌 응답 실패: {exc}")
|
||||||
|
return "미안, 지금 잠깐 생각이 안 났어. 다시 말해줄래?"
|
||||||
|
if not text:
|
||||||
|
return "음, 뭐라고 해야 할지 모르겠어. 다시 말해줄래?"
|
||||||
|
self._history.append((heard, text))
|
||||||
|
if len(self._history) > self._history_turns:
|
||||||
|
self._history = self._history[-self._history_turns:]
|
||||||
|
return text
|
||||||
|
|
||||||
def transcribe_upload(self, audio_bytes: bytes) -> dict:
|
def transcribe_upload(self, audio_bytes: bytes) -> dict:
|
||||||
"""ffmpeg-normalise an uploaded blob to 16 kHz mono wav, transcribe it
|
"""ffmpeg-normalise an uploaded blob to 16 kHz mono wav, transcribe it
|
||||||
on the GPU, and record the result as a monitor turn so it also shows in
|
on the GPU, and record the result as a monitor turn so it also shows in
|
||||||
|
|||||||
Reference in New Issue
Block a user