From e8234b7fb18c207d35298bb5169911ac9cec5e97 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Sat, 13 Jun 2026 22:13:39 +0900 Subject: [PATCH] feat(stt-log): log the WHOLE turn pipeline to the transcript channel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The transcript channel only showed successful transcripts, so dropped utterances (the 47/50 misses) were invisible. Now every captured utterance is mirrored with its outcome and per-stage timing: - too-short blip (<300ms), VAD "음성 아님(VAD 차단)", "인식 실패", "답변 없음", or "ok" - transcript + reply (or "(무응답)") - ⏱️ stt/llm seconds The bridge meta now carries note + stt_sec + think_sec; voice.ts fires onTurn for every turn (not only non-empty transcripts) and for the too-short drop; userbot formats the diagnostic line. Co-Authored-By: Claude Opus 4.7 --- bot/src/bridge.ts | 5 +++++ bot/src/userbot.ts | 39 ++++++++++++++++++++++++++++----------- bot/src/voice.ts | 34 +++++++++++++++++++++++++++------- bridge/server.py | 12 +++++++++--- 4 files changed, 69 insertions(+), 21 deletions(-) diff --git a/bot/src/bridge.ts b/bot/src/bridge.ts index 75a575a..1b5a0a3 100644 --- a/bot/src/bridge.ts +++ b/bot/src/bridge.ts @@ -45,6 +45,11 @@ export interface ConverseMeta { reply: string; error?: string | null; broadcast_action?: BroadcastAction | null; + /** Why this turn produced (or didn't produce) a transcript/reply. */ + note?: string; + /** Per-stage timing (seconds) for diagnosing latency. */ + stt_sec?: number; + think_sec?: number; } export interface ConverseStreamHandlers { diff --git a/bot/src/userbot.ts b/bot/src/userbot.ts index ac34e14..d7984d5 100644 --- a/bot/src/userbot.ts +++ b/bot/src/userbot.ts @@ -34,16 +34,33 @@ async function loadSelfbot(): Promise { } } -/** Mirror a heard voice turn (transcript + reply) into a text channel. */ -async function postTranscript(client: AnyClient, transcript: string, reply: string): Promise { +interface TurnInfo { + transcript: string; + reply: string; + note?: string; + sttSec?: number; + thinkSec?: number; +} + +/** Mirror EVERY heard utterance (and why it did/didn't answer) to a text + * channel, so misses and latency are diagnosable without hearing the bot. */ +async function postTranscript(client: AnyClient, info: TurnInfo): Promise { const chId = config.transcriptChannelId; if (!chId) return; + const timing = + info.sttSec != null || info.thinkSec != null + ? ` ⏱️ stt ${info.sttSec ?? "?"}s · llm ${info.thinkSec ?? "?"}s` + : ""; + let msg: string; + if (info.transcript) { + const r = (info.reply || "").trim(); + msg = `🎤 들음 → 🗣️ "${info.transcript}"\n🤖 답변: ${r || "(무응답)"}${timing}`; + } else { + msg = `🎤 들음 → ❌ ${info.note || "무시됨"}${timing}`; + } try { const ch: any = await client.channels.fetch(chId).catch(() => null); - if (ch?.send) { - const r = (reply || "").trim(); - await ch.send(`🗣️ 들은 말: ${transcript}\n🤖 답변: ${r || "(무응답)"}`); - } + if (ch?.send) await ch.send(msg); } catch (e) { console.error("[userbot] transcript post failed:", e); } @@ -58,11 +75,11 @@ async function joinAndListen(client: AnyClient, channelId: string): Promise { - console.log(`🗣️ ${transcript}\n🤖 ${reply}`); - // Mirror every heard utterance (and the reply, if any) to a text channel so - // you can see what the bot understood even when it doesn't answer aloud. - void postTranscript(client, transcript, reply); + session.onTurn = (info) => { + console.log(`🗣️ ${info.transcript || "(" + (info.note || "empty") + ")"}\n🤖 ${info.reply}`); + // Mirror every heard utterance (and the reply / drop reason) to a text + // channel so you can see what the bot understood even when it doesn't answer. + void postTranscript(client, info); }; // Screen-share mode (STREAM_BROWSER=true): auto-start the broadcast on join, // report its live state to the brain each turn, and let the brain toggle it by diff --git a/bot/src/voice.ts b/bot/src/voice.ts index 5610089..d39a815 100644 --- a/bot/src/voice.ts +++ b/bot/src/voice.ts @@ -69,8 +69,17 @@ export class VoiceSession { /** Pending reply clips. Played one at a time so concurrent speakers don't * cut each other's replies off. */ private playQueue: Buffer[] = []; - /** Optional callback to surface transcripts/replies to a text channel. */ - onTurn?: (info: { user: string; transcript: string; reply: string }) => void; + /** Optional callback to surface EVERY heard utterance (and its outcome) to a + * text channel — including ones dropped before/at STT — so misses are + * diagnosable. `note` says why (e.g. "음성 아님(VAD 차단)", "너무 짧음", "ok"). */ + onTurn?: (info: { + user: string; + transcript: string; + reply: string; + note?: string; + sttSec?: number; + thinkSec?: number; + }) => void; /** Live screen-share state, sent with each turn so the brain routes search * (Chrome while broadcasting, Gemini when off). */ getBroadcasting?: () => boolean; @@ -123,8 +132,12 @@ export class VoiceSession { if (!chunks.length) return; const mono = stereoToMono(Buffer.concat(chunks)); - // Ignore blips shorter than ~300ms (likely noise / key clicks). - if (mono.length < DISCORD_RATE * 0.3 * 2) return; + // Ignore blips shorter than ~300ms (likely noise / key clicks) — but still + // report them so the transcript channel shows every captured utterance. + if (mono.length < DISCORD_RATE * 0.3 * 2) { + this.onTurn?.({ user: userId, transcript: "", reply: "", note: "너무 짧음(<300ms)" }); + return; + } const wav = pcm16MonoToWav(mono, DISCORD_RATE); try { @@ -133,9 +146,16 @@ export class VoiceSession { // so the first sentence starts playing while the rest are still spoken. await converseStream(wav, this.getBroadcasting?.(), { onMeta: async (meta) => { - if (meta.transcript) { - this.onTurn?.({ user: userId, transcript: meta.transcript, reply: meta.reply }); - } + // Report EVERY turn (even empty/VAD-dropped) so the transcript channel + // explains why a turn did or didn't answer. + this.onTurn?.({ + user: userId, + transcript: meta.transcript, + reply: meta.reply, + note: meta.note, + sttSec: meta.stt_sec, + thinkSec: meta.think_sec, + }); // Apply any broadcast directive the brain requested (e.g. user said // "방송 켜줘 / 꺼줘") before the reply audio plays. The meta line // always precedes the audio clips, so awaiting here preserves order. diff --git a/bridge/server.py b/bridge/server.py index 4a54172..5c86794 100644 --- a/bridge/server.py +++ b/bridge/server.py @@ -211,7 +211,7 @@ def transcribe(wav_bytes: bytes) -> dict: log=lambda m: print(f"[bridge] {m}", flush=True), ): print("[bridge] no speech detected (VAD) — skipping STT", flush=True) - return {"text": "", "language": None} + return {"text": "", "language": None, "note": "음성 아님(VAD 차단)"} segments, info = _whisper.transcribe(audio, beam_size=1, language=STT_LANGUAGE) # Second line of defence: drop non-speech / hallucinated segments by @@ -231,7 +231,8 @@ def transcribe(wav_bytes: bytes) -> dict: log=lambda m: print(f"[bridge] {m}", flush=True), ) text = "".join(seg.text for seg in kept).strip() - return {"text": text, "language": getattr(info, "language", None)} + note = "ok" if text else "인식 실패(빈 결과/필터)" + return {"text": text, "language": getattr(info, "language", None), "note": note} def think(text: str, language: Optional[str] = None, broadcasting: Optional[bool] = None) -> dict: @@ -458,7 +459,9 @@ def http_converse_stream(): transcript = stt.get("text", "") if not transcript: yield json.dumps({"type": "meta", "transcript": "", "language": stt.get("language"), - "reply": "", "error": stt.get("error"), "broadcast_action": None}) + "\n" + "reply": "", "error": stt.get("error"), + "note": stt.get("note", "빈 결과"), + "stt_sec": round(t_stt - t0, 1), "broadcast_action": None}) + "\n" yield json.dumps({"type": "end"}) + "\n" return result = think(transcript, stt.get("language"), broadcasting) @@ -470,6 +473,9 @@ def http_converse_stream(): "language": stt.get("language"), "reply": reply, "error": result.get("error"), + "note": "ok" if reply.strip() else "답변 없음", + "stt_sec": round(t_stt - t0, 1), + "think_sec": round(t_think - t_stt, 1), "broadcast_action": result.get("broadcast_action"), }) + "\n" tts_total = 0.0