From 6d72e10f9cc86c7faf9af5c082d1fd93adf80714 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Sat, 13 Jun 2026 14:53:54 +0900 Subject: [PATCH] feat(bridge): gate Whisper behind Silero VAD; harden broadcast auto-start Address review of the noise/broadcast fixes: - STT now refuses to run Whisper on non-speech. transcribe() runs the Silero VAD (bundled with faster-whisper, no new dep) BEFORE the model, so noise or a brief loud blip with no real speech never reaches STT and can't be hallucinated into a transcript. The no_speech_prob/avg_logprob post-filter stays as a second line of defence (a clap the VAD lets through is still killed by Whisper's own no_speech_prob). VAD is env-tunable (VAD_THRESHOLD, VAD_MIN_SPEECH_MS, VAD_ENABLED) and fail-open so a VAD error never swallows a real utterance. Validated on real audio: synthesised Korean speech passes; silence, a 50ms blip and white noise are rejected. - Broadcast auto-start no longer blocks the voice join and no longer silently swallows failures: wiring is synchronous, the Go-Live start runs in the background with a bounded retry and a loud final-failure log. Co-Authored-By: Claude Opus 4.7 --- bot/src/broadcast.test.ts | 81 +++++++++++++++++++++------------ bot/src/broadcast.ts | 60 ++++++++++++++++++------ bridge/server.py | 34 ++++++++++++-- bridge/stt_filter.py | 43 +++++++++++++++++ tests/test_bridge_stt_filter.py | 24 ++++++++++ 5 files changed, 194 insertions(+), 48 deletions(-) diff --git a/bot/src/broadcast.test.ts b/bot/src/broadcast.test.ts index 92e8c2b..191c5d4 100644 --- a/bot/src/broadcast.test.ts +++ b/bot/src/broadcast.test.ts @@ -3,7 +3,7 @@ import { test, expect } from "bun:test"; // broadcast.ts pulls in the runtime `config`, which requires DISCORD_GUILD_ID. // Set it before the dynamic import so the module loads without a real .env. process.env.DISCORD_GUILD_ID ||= "test-guild"; -const { coupleBroadcast } = await import("./broadcast.ts"); +const { wireBroadcast, autoStartBroadcast } = await import("./broadcast.ts"); /** Minimal in-memory ScreenStreamer that records start/stop calls. */ function fakeStreamer(active = false) { @@ -27,44 +27,67 @@ function fakeStreamer(active = false) { const ctx = { guildId: "g", voiceChannelId: "c" }; -test("auto-starts the broadcast when joining with the stream idle", async () => { +test("wireBroadcast reports live state to the brain", () => { const s = fakeStreamer(false); const session: any = {}; - await coupleBroadcast(session, s as any, ctx); - expect(s.calls).toEqual(["start"]); - expect(session.getBroadcasting()).toBe(true); -}); - -test("does not restart a broadcast that is already live", async () => { - const s = fakeStreamer(true); - const session: any = {}; - await coupleBroadcast(session, s as any, ctx); - expect(s.calls).toEqual([]); - expect(session.getBroadcasting()).toBe(true); -}); - -test("the brain can toggle the stream by voice via onBroadcastAction", async () => { - const s = fakeStreamer(false); - const session: any = {}; - await coupleBroadcast(session, s as any, ctx); // auto-start fires - await session.onBroadcastAction("start"); // already live -> no-op - await session.onBroadcastAction("stop"); // stops + wireBroadcast(session, s as any, ctx); expect(session.getBroadcasting()).toBe(false); - await session.onBroadcastAction("start"); // restarts - expect(session.getBroadcasting()).toBe(true); - expect(s.calls).toEqual(["start", "stop", "start"]); }); -test("an auto-start failure is swallowed but the toggle hooks are still wired", async () => { +test("wireBroadcast lets the brain toggle the stream by voice", async () => { + const s = fakeStreamer(false); + const session: any = {}; + wireBroadcast(session, s as any, ctx); + await session.onBroadcastAction("start"); // idle -> start + expect(session.getBroadcasting()).toBe(true); + await session.onBroadcastAction("start"); // already live -> no-op + await session.onBroadcastAction("stop"); // stop + expect(session.getBroadcasting()).toBe(false); + expect(s.calls).toEqual(["start", "stop"]); +}); + +test("autoStartBroadcast starts the broadcast when idle", async () => { + const s = fakeStreamer(false); + const ok = await autoStartBroadcast(s as any, ctx, 2, 0); + expect(ok).toBe(true); + expect(s.calls).toEqual(["start"]); +}); + +test("autoStartBroadcast does not restart an already-live broadcast", async () => { + const s = fakeStreamer(true); + const ok = await autoStartBroadcast(s as any, ctx, 2, 0); + expect(ok).toBe(true); + expect(s.calls).toEqual([]); +}); + +test("autoStartBroadcast retries a transient failure and then succeeds", async () => { + let on = false; + const calls: string[] = []; + const s: any = { + isActive: () => on, + async start() { + calls.push("start"); + if (calls.length === 1) throw new Error("transient: voice not ready"); + on = true; + }, + async stop() {}, + }; + const ok = await autoStartBroadcast(s, ctx, 2, 0); + expect(ok).toBe(true); + expect(calls).toEqual(["start", "start"]); +}); + +test("autoStartBroadcast returns false after exhausting retries (no throw)", async () => { + let attempts = 0; const s: any = { isActive: () => false, async start() { + attempts++; throw new Error("boom"); }, async stop() {}, }; - const session: any = {}; - await coupleBroadcast(session, s, ctx); - expect(typeof session.getBroadcasting).toBe("function"); - expect(typeof session.onBroadcastAction).toBe("function"); + const ok = await autoStartBroadcast(s, ctx, 2, 0); + expect(ok).toBe(false); + expect(attempts).toBe(2); // retried, then gave up without throwing }); diff --git a/bot/src/broadcast.ts b/bot/src/broadcast.ts index 598ebc9..0fc6ded 100644 --- a/bot/src/broadcast.ts +++ b/bot/src/broadcast.ts @@ -34,20 +34,16 @@ export function peekStreamer(guildId: string): ScreenStreamer | undefined { type BroadcastSession = Pick; /** - * Wire a voice session to a streamer and auto-start the broadcast. Pure (no - * config / registry access) so it can be unit-tested with a fake streamer: + * Wire a voice session to a streamer (no auto-start). Pure (no config / registry + * access) so it can be unit-tested with a fake streamer: * - report live state to the brain each turn, - * - let the brain toggle the stream by voice, - * - auto-start on join (skipping if already live). - * - * Auto-start failures are swallowed so a broadcast problem never blocks the - * voice conversation from coming up. + * - let the brain toggle the stream by voice. */ -export async function coupleBroadcast( +export function wireBroadcast( session: BroadcastSession, streamer: ScreenStreamer, ctx: StreamContext, -): Promise { +): void { session.getBroadcasting = () => streamer.isActive(); session.onBroadcastAction = async (action) => { if (action === "start") { @@ -56,16 +52,51 @@ export async function coupleBroadcast( await streamer.stop(); } }; - try { - if (!streamer.isActive()) await streamer.start(ctx); - } catch (e) { - console.error("[broadcast] auto-start failed:", e); +} + +/** + * Auto-start the broadcast, retrying a couple of times before giving up. The + * selfbot Go-Live path takes ~10-15s of humanised delays and can fail on a + * transient (login race, voice not ready yet), so a bounded retry recovers the + * common case. On final failure it logs loudly rather than silently leaving the + * user in voice with no broadcast. Never throws: a broadcast problem must not + * tear down the voice conversation. Returns whether the stream ended up live. + */ +export async function autoStartBroadcast( + streamer: ScreenStreamer, + ctx: StreamContext, + attempts = 2, + retryDelayMs = 1500, +): Promise { + for (let attempt = 1; attempt <= attempts; attempt++) { + try { + if (streamer.isActive()) return true; + await streamer.start(ctx); + if (streamer.isActive()) return true; + console.error( + `[broadcast] auto-start attempt ${attempt}/${attempts} did not go live`, + ); + } catch (e) { + console.error(`[broadcast] auto-start attempt ${attempt}/${attempts} failed:`, e); + } + if (attempt < attempts && retryDelayMs > 0) { + await new Promise((r) => setTimeout(r, retryDelayMs)); + } } + console.error( + "[broadcast] ⚠️ auto-start FAILED after retries — voice is up but NOT broadcasting " + + "(STREAM_BROWSER=true). Trigger it by voice ('방송 켜줘') or check the selfbot stream deps.", + ); + return false; } /** * Couple the broadcast to a freshly joined voice session when screen-share mode * is on (STREAM_BROWSER=true). No-op in voice-only mode (STREAM_BROWSER=false). + * + * The toggle hooks are wired synchronously; the auto-start runs in the + * BACKGROUND so the ~10-15s Go-Live handshake never blocks the voice session + * from coming up. The bot can already hear and reply while the stream warms. */ export async function maybeCoupleBroadcast( session: BroadcastSession, @@ -73,7 +104,8 @@ export async function maybeCoupleBroadcast( ): Promise { if (!config.screenBrowser) return; const streamer = await getStreamer(ctx.guildId); - await coupleBroadcast(session, streamer, ctx); + wireBroadcast(session, streamer, ctx); + void autoStartBroadcast(streamer, ctx); } /** Stop the broadcast for a guild if one is live (e.g. on leave). */ diff --git a/bridge/server.py b/bridge/server.py index f1cc9be..55827a1 100644 --- a/bridge/server.py +++ b/bridge/server.py @@ -51,10 +51,10 @@ from flask import Flask, request, jsonify, Response, stream_with_context try: # package-relative when imported as ``bridge.server`` from bridge.text_utils import split_sentences - from bridge.stt_filter import filter_speech_segments + from bridge.stt_filter import filter_speech_segments, has_speech except ImportError: # script-relative when run as ``bridge/server.py`` from text_utils import split_sentences - from stt_filter import filter_speech_segments + from stt_filter import filter_speech_segments, has_speech app = Flask(__name__) @@ -66,6 +66,15 @@ BRIDGE_PORT = int(os.environ.get("BRIDGE_PORT", "8765")) BRAIN_ENABLED = os.environ.get("JARVIS_BRAIN_ENABLED", "1") not in ("0", "false", "False") TTS_ENABLED = os.environ.get("JARVIS_TTS_ENABLED", "1") not in ("0", "false", "False") +# Pre-STT speech gate (Silero VAD). Tunable for the Discord mic without a code +# change: raise VAD_THRESHOLD to reject more noise, lower it to catch quieter +# speech. VAD_MIN_SPEECH_MS is the shortest run of speech that counts (a brief +# loud blip shorter than this never reaches Whisper). Set VAD_ENABLED=0 to fall +# back to the old behaviour (always transcribe, rely on the post-filter only). +VAD_ENABLED = os.environ.get("VAD_ENABLED", "1") not in ("0", "false", "False") +VAD_THRESHOLD = float(os.environ.get("VAD_THRESHOLD", "0.4")) +VAD_MIN_SPEECH_MS = int(os.environ.get("VAD_MIN_SPEECH_MS", "200")) + # TTS engine: "melo" (MeloTTS Korean speaker, the warm worker) is the primary # voice; Piper is kept as a fallback if the worker is unreachable. Set # TTS_ENGINE=piper to disable MeloTTS entirely. @@ -184,10 +193,25 @@ def transcribe(wav_bytes: bytes) -> dict: x_old = np.linspace(0.0, 1.0, num=audio.size, endpoint=False) x_new = np.linspace(0.0, 1.0, num=n_out, endpoint=False) audio = np.interp(x_new, x_old, audio).astype(np.float32) + # Pre-STT speech gate: don't even invoke Whisper unless there is real speech + # in the clip. Noise or a brief loud blip (no actual speech) is dropped here, + # before transcription, so the model never gets a chance to hallucinate a + # phrase from it. Fail-open inside has_speech() keeps a real utterance from + # being swallowed if the VAD is unavailable. + if VAD_ENABLED and not has_speech( + audio, + 16000, + threshold=VAD_THRESHOLD, + min_speech_duration_ms=VAD_MIN_SPEECH_MS, + log=lambda m: print(f"[bridge] {m}", flush=True), + ): + print("[bridge] no speech detected (VAD) — skipping STT", flush=True) + return {"text": "", "language": None} + segments, info = _whisper.transcribe(audio, beam_size=1) - # Speech gate: drop non-speech / hallucinated segments so a brief loud sound - # or background noise (mic blip with no real speech) does not become a - # transcript and make the bot reply to nothing. Mirrors the desktop + # Second line of defence: even speech-like audio (e.g. a clap the VAD let + # through) can make Whisper hallucinate, so drop non-speech / low-confidence + # segments by Whisper's own no_speech_prob + avg_logprob. Mirrors the desktop # listener's policy, driven by the same config thresholds. no_speech_threshold = getattr(_cfg, "whisper_no_speech_threshold", 0.5) min_confidence = getattr(_cfg, "whisper_min_confidence", 0.3) diff --git a/bridge/stt_filter.py b/bridge/stt_filter.py index 7e36ba1..ae7a22f 100644 --- a/bridge/stt_filter.py +++ b/bridge/stt_filter.py @@ -25,6 +25,49 @@ from __future__ import annotations from typing import Callable, Optional +def has_speech( + audio, + sampling_rate: int = 16000, + *, + threshold: float = 0.4, + min_speech_duration_ms: int = 200, + min_silence_duration_ms: int = 100, + log: Optional[Callable[[str], None]] = None, +) -> bool: + """Pre-STT speech gate: ``True`` only if there is at least one real speech + region in ``audio`` (16 kHz mono float32). + + This runs BEFORE Whisper so the model is never invoked on pure noise or a + brief loud blip (a clap, a key clack, a mic pop) that momentarily opened the + voice gate without anyone speaking. It uses the Silero VAD bundled with + faster-whisper (no extra dependency). The threshold is deliberately a little + below the faster-whisper default (0.5) so quiet but real speech is not + dropped; precision against confident noise-hallucinations is provided by the + downstream ``filter_speech_segments`` no_speech_prob gate. + + Fail-open: if the VAD is unavailable or errors, return ``True`` so STT still + runs rather than silently swallowing a real utterance. + """ + try: + from faster_whisper.vad import get_speech_timestamps, VadOptions + except Exception: # VAD not available in this build -> don't block STT + return True + try: + if getattr(audio, "size", 1) == 0: + return False + opts = VadOptions( + threshold=threshold, + min_speech_duration_ms=min_speech_duration_ms, + min_silence_duration_ms=min_silence_duration_ms, + ) + timestamps = get_speech_timestamps(audio, opts, sampling_rate) + return len(timestamps) > 0 + except Exception as e: # pragma: no cover - defensive + if log: + log(f"VAD check failed, falling back to STT: {e}") + return True + + def is_non_speech(no_speech_prob: float, threshold: float) -> bool: """True when Whisper flags a segment as non-speech (``>= threshold``).""" return no_speech_prob >= threshold diff --git a/tests/test_bridge_stt_filter.py b/tests/test_bridge_stt_filter.py index b99e613..7b90dbd 100644 --- a/tests/test_bridge_stt_filter.py +++ b/tests/test_bridge_stt_filter.py @@ -7,15 +7,39 @@ produces no transcript and no reply. Thresholds are config-driven, so the tests pass explicit references rather than hardcoding the production defaults. """ +import numpy as np import pytest from bridge.stt_filter import ( filter_speech_segments, + has_speech, is_non_speech, segment_confidence, ) +@pytest.mark.unit +def test_has_speech_rejects_silence(): + # Pure digital silence is unambiguously non-speech: VAD must skip STT. + silence = np.zeros(16000, dtype=np.float32) # 1s @ 16kHz + assert has_speech(silence, 16000) is False + + +@pytest.mark.unit +def test_has_speech_rejects_a_brief_loud_blip(): + # A short loud transient (a clap / pop): mostly silence with a 50ms spike, + # below the min-speech duration, so no real speech region is found. + audio = np.zeros(16000, dtype=np.float32) + audio[8000:8800] = 0.9 # ~50ms full-scale burst + assert has_speech(audio, 16000, min_speech_duration_ms=200) is False + + +@pytest.mark.unit +def test_has_speech_fails_open_on_empty_when_vad_present_returns_false(): + # Empty audio has nothing to transcribe; treat as non-speech. + assert has_speech(np.zeros(0, dtype=np.float32), 16000) is False + + class Seg: """Minimal stand-in for a faster-whisper segment."""