feat(voice): real Claude brain in the Discord loop (think, not echo)
The voice loop echoed the recognised text. Wire the real brain: the Discord voice-turn now runs STT -> ClaudeBrain.respond (with rolling conversation history) -> TTS, so the bot actually thinks and answers. --voice-server builds the brain by default (WSAI_BRAIN=claude, WSAI_BRAIN_MODEL overridable) and gracefully falls back to echo if anthropic/Claude auth is unavailable. A brain error speaks a short apology instead of killing the loop. Verified end-to-end: an utterance wav returns X-Heard plus a distinct Claude X-Reply and a synthesised reply wav on device=cuda. 12 tests pass. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -172,12 +172,15 @@ class Dashboard:
|
||||
"""
|
||||
|
||||
def __init__(self, monitor: Monitor, host: str = "0.0.0.0", port: int = 8787,
|
||||
stt=None, tts=None) -> None:
|
||||
stt=None, tts=None, brain=None, history_turns: int = 12) -> None:
|
||||
self.monitor = monitor
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.stt = stt
|
||||
self.tts = tts
|
||||
self.brain = brain
|
||||
self._history: list[tuple[str, str]] = []
|
||||
self._history_turns = history_turns
|
||||
self._server: ThreadingHTTPServer | None = None
|
||||
self._thread: threading.Thread | None = None
|
||||
self._loop = None
|
||||
@@ -230,9 +233,9 @@ class Dashboard:
|
||||
|
||||
def voice_turn(self, audio_bytes: bytes) -> dict:
|
||||
"""One Discord voice turn: decode the uploaded utterance, recognise it
|
||||
on the GPU, produce a reply (echo of what was heard for now), synthesise
|
||||
it on the GPU, and return {heard, reply, wav} where wav is the reply
|
||||
audio bytes for the bot to play back into the channel."""
|
||||
on the GPU, think of a reply (Claude brain if wired, else echo),
|
||||
synthesise it on the GPU, and return {heard, reply, wav} where wav is the
|
||||
reply audio bytes for the bot to play back into the channel."""
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
@@ -253,19 +256,19 @@ class Dashboard:
|
||||
["ffmpeg", "-y", "-i", src, "-ar", "16000", "-ac", "1", wav],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
heard = self._submit(self.stt.transcribe(wav)) or ""
|
||||
heard = (self._submit(self.stt.transcribe(wav)) or "").strip()
|
||||
turn.heard(heard or "(빈 결과)")
|
||||
reply_text = heard.strip()
|
||||
if not reply_text:
|
||||
# Nothing recognised (silence/noise): skip TTS, tell the bot.
|
||||
if not heard:
|
||||
# Nothing recognised (silence/noise): skip the turn, tell the bot.
|
||||
turn.finish()
|
||||
return {"heard": heard, "reply": "", "wav": b""}
|
||||
reply_text = self._think(heard)
|
||||
turn.replied(reply_text)
|
||||
out_path = self._submit(self.tts.synth(reply_text))
|
||||
with open(out_path, "rb") as f:
|
||||
reply_wav = f.read()
|
||||
ms = int((time.monotonic() - t0) * 1000)
|
||||
turn.replied(reply_text)
|
||||
step = turn.step("STT+TTS(GPU)")
|
||||
step = turn.step("STT+두뇌+TTS" if self.brain else "STT+TTS(GPU)")
|
||||
step.ok, step.ms = True, float(ms)
|
||||
turn._steps.append(step)
|
||||
turn.finish()
|
||||
@@ -288,6 +291,27 @@ class Dashboard:
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _think(self, heard: str) -> str:
|
||||
"""Turn what was heard into a reply. Uses the Claude brain when wired
|
||||
(with rolling conversation history); falls back to echo if there is no
|
||||
brain, and to a spoken apology if the brain call fails — so one API hiccup
|
||||
never kills the voice loop."""
|
||||
if self.brain is None:
|
||||
return heard # echo mode
|
||||
try:
|
||||
reply = self._submit(self.brain.respond(heard, None, list(self._history)))
|
||||
text = (reply.text or "").strip()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.exception("brain failed")
|
||||
self.monitor.log("error", f"두뇌 응답 실패: {exc}")
|
||||
return "미안, 지금 잠깐 생각이 안 났어. 다시 말해줄래?"
|
||||
if not text:
|
||||
return "음, 뭐라고 해야 할지 모르겠어. 다시 말해줄래?"
|
||||
self._history.append((heard, text))
|
||||
if len(self._history) > self._history_turns:
|
||||
self._history = self._history[-self._history_turns:]
|
||||
return text
|
||||
|
||||
def transcribe_upload(self, audio_bytes: bytes) -> dict:
|
||||
"""ffmpeg-normalise an uploaded blob to 16 kHz mono wav, transcribe it
|
||||
on the GPU, and record the result as a monitor turn so it also shows in
|
||||
|
||||
Reference in New Issue
Block a user