feat(bridge): gate Whisper behind Silero VAD; harden broadcast auto-start

Address review of the noise/broadcast fixes:

- STT now refuses to run Whisper on non-speech. transcribe() runs the Silero
  VAD (bundled with faster-whisper, no new dep) BEFORE the model, so noise or a
  brief loud blip with no real speech never reaches STT and can't be
  hallucinated into a transcript. The no_speech_prob/avg_logprob post-filter
  stays as a second line of defence (a clap the VAD lets through is still killed
  by Whisper's own no_speech_prob). VAD is env-tunable (VAD_THRESHOLD,
  VAD_MIN_SPEECH_MS, VAD_ENABLED) and fail-open so a VAD error never swallows a
  real utterance. Validated on real audio: synthesised Korean speech passes;
  silence, a 50ms blip and white noise are rejected.

- Broadcast auto-start no longer blocks the voice join and no longer silently
  swallows failures: wiring is synchronous, the Go-Live start runs in the
  background with a bounded retry and a loud final-failure log.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

bot/src/broadcast.test.ts

@@ -3,7 +3,7 @@ import { test, expect } from "bun:test";
 // broadcast.ts pulls in the runtime `config`, which requires DISCORD_GUILD_ID.
 // Set it before the dynamic import so the module loads without a real .env.
 process.env.DISCORD_GUILD_ID ||= "test-guild";
-const { coupleBroadcast } = await import("./broadcast.ts");
+const { wireBroadcast, autoStartBroadcast } = await import("./broadcast.ts");
 
 /** Minimal in-memory ScreenStreamer that records start/stop calls. */
 function fakeStreamer(active = false) {
@@ -27,44 +27,67 @@ function fakeStreamer(active = false) {
 
 const ctx = { guildId: "g", voiceChannelId: "c" };
 
-test("auto-starts the broadcast when joining with the stream idle", async () => {
+test("wireBroadcast reports live state to the brain", () => {
   const s = fakeStreamer(false);
   const session: any = {};
-  await coupleBroadcast(session, s as any, ctx);
+  wireBroadcast(session, s as any, ctx);
-  expect(s.calls).toEqual(["start"]);
-  expect(session.getBroadcasting()).toBe(true);
-});
-
-test("does not restart a broadcast that is already live", async () => {
-  const s = fakeStreamer(true);
-  const session: any = {};
-  await coupleBroadcast(session, s as any, ctx);
-  expect(s.calls).toEqual([]);
-  expect(session.getBroadcasting()).toBe(true);
-});
-
-test("the brain can toggle the stream by voice via onBroadcastAction", async () => {
-  const s = fakeStreamer(false);
-  const session: any = {};
-  await coupleBroadcast(session, s as any, ctx); // auto-start fires
-  await session.onBroadcastAction("start"); // already live -> no-op
-  await session.onBroadcastAction("stop"); // stops
   expect(session.getBroadcasting()).toBe(false);
-  await session.onBroadcastAction("start"); // restarts
-  expect(session.getBroadcasting()).toBe(true);
-  expect(s.calls).toEqual(["start", "stop", "start"]);
 });
 
-test("an auto-start failure is swallowed but the toggle hooks are still wired", async () => {
+test("wireBroadcast lets the brain toggle the stream by voice", async () => {
+  const s = fakeStreamer(false);
+  const session: any = {};
+  wireBroadcast(session, s as any, ctx);
+  await session.onBroadcastAction("start"); // idle -> start
+  expect(session.getBroadcasting()).toBe(true);
+  await session.onBroadcastAction("start"); // already live -> no-op
+  await session.onBroadcastAction("stop"); // stop
+  expect(session.getBroadcasting()).toBe(false);
+  expect(s.calls).toEqual(["start", "stop"]);
+});
+
+test("autoStartBroadcast starts the broadcast when idle", async () => {
+  const s = fakeStreamer(false);
+  const ok = await autoStartBroadcast(s as any, ctx, 2, 0);
+  expect(ok).toBe(true);
+  expect(s.calls).toEqual(["start"]);
+});
+
+test("autoStartBroadcast does not restart an already-live broadcast", async () => {
+  const s = fakeStreamer(true);
+  const ok = await autoStartBroadcast(s as any, ctx, 2, 0);
+  expect(ok).toBe(true);
+  expect(s.calls).toEqual([]);
+});
+
+test("autoStartBroadcast retries a transient failure and then succeeds", async () => {
+  let on = false;
+  const calls: string[] = [];
+  const s: any = {
+    isActive: () => on,
+    async start() {
+      calls.push("start");
+      if (calls.length === 1) throw new Error("transient: voice not ready");
+      on = true;
+    },
+    async stop() {},
+  };
+  const ok = await autoStartBroadcast(s, ctx, 2, 0);
+  expect(ok).toBe(true);
+  expect(calls).toEqual(["start", "start"]);
+});
+
+test("autoStartBroadcast returns false after exhausting retries (no throw)", async () => {
+  let attempts = 0;
   const s: any = {
     isActive: () => false,
     async start() {
+      attempts++;
       throw new Error("boom");
     },
     async stop() {},
   };
-  const session: any = {};
+  const ok = await autoStartBroadcast(s, ctx, 2, 0);
-  await coupleBroadcast(session, s, ctx);
+  expect(ok).toBe(false);
-  expect(typeof session.getBroadcasting).toBe("function");
+  expect(attempts).toBe(2); // retried, then gave up without throwing
-  expect(typeof session.onBroadcastAction).toBe("function");
 });

bot/src/broadcast.ts

@@ -34,20 +34,16 @@ export function peekStreamer(guildId: string): ScreenStreamer | undefined {
 type BroadcastSession = Pick<VoiceSession, "getBroadcasting" | "onBroadcastAction">;
 
 /**
- * Wire a voice session to a streamer and auto-start the broadcast. Pure (no
+ * Wire a voice session to a streamer (no auto-start). Pure (no config / registry
- * config / registry access) so it can be unit-tested with a fake streamer:
+ * access) so it can be unit-tested with a fake streamer:
  *  - report live state to the brain each turn,
- *  - let the brain toggle the stream by voice,
+ *  - let the brain toggle the stream by voice.
- *  - auto-start on join (skipping if already live).
- *
- * Auto-start failures are swallowed so a broadcast problem never blocks the
- * voice conversation from coming up.
  */
-export async function coupleBroadcast(
+export function wireBroadcast(
   session: BroadcastSession,
   streamer: ScreenStreamer,
   ctx: StreamContext,
-): Promise<void> {
+): void {
   session.getBroadcasting = () => streamer.isActive();
   session.onBroadcastAction = async (action) => {
     if (action === "start") {
@@ -56,16 +52,51 @@ export async function coupleBroadcast(
       await streamer.stop();
     }
   };
-  try {
+}
-    if (!streamer.isActive()) await streamer.start(ctx);
+
-  } catch (e) {
+/**
-    console.error("[broadcast] auto-start failed:", e);
+ * Auto-start the broadcast, retrying a couple of times before giving up. The
+ * selfbot Go-Live path takes ~10-15s of humanised delays and can fail on a
+ * transient (login race, voice not ready yet), so a bounded retry recovers the
+ * common case. On final failure it logs loudly rather than silently leaving the
+ * user in voice with no broadcast. Never throws: a broadcast problem must not
+ * tear down the voice conversation. Returns whether the stream ended up live.
+ */
+export async function autoStartBroadcast(
+  streamer: ScreenStreamer,
+  ctx: StreamContext,
+  attempts = 2,
+  retryDelayMs = 1500,
+): Promise<boolean> {
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    try {
+      if (streamer.isActive()) return true;
+      await streamer.start(ctx);
+      if (streamer.isActive()) return true;
+      console.error(
+        `[broadcast] auto-start attempt ${attempt}/${attempts} did not go live`,
+      );
+    } catch (e) {
+      console.error(`[broadcast] auto-start attempt ${attempt}/${attempts} failed:`, e);
+    }
+    if (attempt < attempts && retryDelayMs > 0) {
+      await new Promise((r) => setTimeout(r, retryDelayMs));
+    }
   }
+  console.error(
+    "[broadcast] ⚠️ auto-start FAILED after retries — voice is up but NOT broadcasting " +
+      "(STREAM_BROWSER=true). Trigger it by voice ('방송 켜줘') or check the selfbot stream deps.",
+  );
+  return false;
 }
 
 /**
  * Couple the broadcast to a freshly joined voice session when screen-share mode
  * is on (STREAM_BROWSER=true). No-op in voice-only mode (STREAM_BROWSER=false).
+ *
+ * The toggle hooks are wired synchronously; the auto-start runs in the
+ * BACKGROUND so the ~10-15s Go-Live handshake never blocks the voice session
+ * from coming up. The bot can already hear and reply while the stream warms.
  */
 export async function maybeCoupleBroadcast(
   session: BroadcastSession,
@@ -73,7 +104,8 @@ export async function maybeCoupleBroadcast(
 ): Promise<void> {
   if (!config.screenBrowser) return;
   const streamer = await getStreamer(ctx.guildId);
-  await coupleBroadcast(session, streamer, ctx);
+  wireBroadcast(session, streamer, ctx);
+  void autoStartBroadcast(streamer, ctx);
 }
 
 /** Stop the broadcast for a guild if one is live (e.g. on leave). */

bridge/server.py

@@ -51,10 +51,10 @@ from flask import Flask, request, jsonify, Response, stream_with_context
 
 try:  # package-relative when imported as ``bridge.server``
     from bridge.text_utils import split_sentences
-    from bridge.stt_filter import filter_speech_segments
+    from bridge.stt_filter import filter_speech_segments, has_speech
 except ImportError:  # script-relative when run as ``bridge/server.py``
     from text_utils import split_sentences
-    from stt_filter import filter_speech_segments
+    from stt_filter import filter_speech_segments, has_speech
 
 app = Flask(__name__)
 
@@ -66,6 +66,15 @@ BRIDGE_PORT = int(os.environ.get("BRIDGE_PORT", "8765"))
 BRAIN_ENABLED = os.environ.get("JARVIS_BRAIN_ENABLED", "1") not in ("0", "false", "False")
 TTS_ENABLED = os.environ.get("JARVIS_TTS_ENABLED", "1") not in ("0", "false", "False")
 
+# Pre-STT speech gate (Silero VAD). Tunable for the Discord mic without a code
+# change: raise VAD_THRESHOLD to reject more noise, lower it to catch quieter
+# speech. VAD_MIN_SPEECH_MS is the shortest run of speech that counts (a brief
+# loud blip shorter than this never reaches Whisper). Set VAD_ENABLED=0 to fall
+# back to the old behaviour (always transcribe, rely on the post-filter only).
+VAD_ENABLED = os.environ.get("VAD_ENABLED", "1") not in ("0", "false", "False")
+VAD_THRESHOLD = float(os.environ.get("VAD_THRESHOLD", "0.4"))
+VAD_MIN_SPEECH_MS = int(os.environ.get("VAD_MIN_SPEECH_MS", "200"))
+
 # TTS engine: "melo" (MeloTTS Korean speaker, the warm worker) is the primary
 # voice; Piper is kept as a fallback if the worker is unreachable. Set
 # TTS_ENGINE=piper to disable MeloTTS entirely.
@@ -184,10 +193,25 @@ def transcribe(wav_bytes: bytes) -> dict:
             x_old = np.linspace(0.0, 1.0, num=audio.size, endpoint=False)
             x_new = np.linspace(0.0, 1.0, num=n_out, endpoint=False)
             audio = np.interp(x_new, x_old, audio).astype(np.float32)
+    # Pre-STT speech gate: don't even invoke Whisper unless there is real speech
+    # in the clip. Noise or a brief loud blip (no actual speech) is dropped here,
+    # before transcription, so the model never gets a chance to hallucinate a
+    # phrase from it. Fail-open inside has_speech() keeps a real utterance from
+    # being swallowed if the VAD is unavailable.
+    if VAD_ENABLED and not has_speech(
+        audio,
+        16000,
+        threshold=VAD_THRESHOLD,
+        min_speech_duration_ms=VAD_MIN_SPEECH_MS,
+        log=lambda m: print(f"[bridge] {m}", flush=True),
+    ):
+        print("[bridge] no speech detected (VAD) — skipping STT", flush=True)
+        return {"text": "", "language": None}
+
     segments, info = _whisper.transcribe(audio, beam_size=1)
-    # Speech gate: drop non-speech / hallucinated segments so a brief loud sound
+    # Second line of defence: even speech-like audio (e.g. a clap the VAD let
-    # or background noise (mic blip with no real speech) does not become a
+    # through) can make Whisper hallucinate, so drop non-speech / low-confidence
-    # transcript and make the bot reply to nothing. Mirrors the desktop
+    # segments by Whisper's own no_speech_prob + avg_logprob. Mirrors the desktop
     # listener's policy, driven by the same config thresholds.
     no_speech_threshold = getattr(_cfg, "whisper_no_speech_threshold", 0.5)
     min_confidence = getattr(_cfg, "whisper_min_confidence", 0.3)

bridge/stt_filter.py

@@ -25,6 +25,49 @@ from __future__ import annotations
 from typing import Callable, Optional
 
 
+def has_speech(
+    audio,
+    sampling_rate: int = 16000,
+    *,
+    threshold: float = 0.4,
+    min_speech_duration_ms: int = 200,
+    min_silence_duration_ms: int = 100,
+    log: Optional[Callable[[str], None]] = None,
+) -> bool:
+    """Pre-STT speech gate: ``True`` only if there is at least one real speech
+    region in ``audio`` (16 kHz mono float32).
+
+    This runs BEFORE Whisper so the model is never invoked on pure noise or a
+    brief loud blip (a clap, a key clack, a mic pop) that momentarily opened the
+    voice gate without anyone speaking. It uses the Silero VAD bundled with
+    faster-whisper (no extra dependency). The threshold is deliberately a little
+    below the faster-whisper default (0.5) so quiet but real speech is not
+    dropped; precision against confident noise-hallucinations is provided by the
+    downstream ``filter_speech_segments`` no_speech_prob gate.
+
+    Fail-open: if the VAD is unavailable or errors, return ``True`` so STT still
+    runs rather than silently swallowing a real utterance.
+    """
+    try:
+        from faster_whisper.vad import get_speech_timestamps, VadOptions
+    except Exception:  # VAD not available in this build -> don't block STT
+        return True
+    try:
+        if getattr(audio, "size", 1) == 0:
+            return False
+        opts = VadOptions(
+            threshold=threshold,
+            min_speech_duration_ms=min_speech_duration_ms,
+            min_silence_duration_ms=min_silence_duration_ms,
+        )
+        timestamps = get_speech_timestamps(audio, opts, sampling_rate)
+        return len(timestamps) > 0
+    except Exception as e:  # pragma: no cover - defensive
+        if log:
+            log(f"VAD check failed, falling back to STT: {e}")
+        return True
+
+
 def is_non_speech(no_speech_prob: float, threshold: float) -> bool:
     """True when Whisper flags a segment as non-speech (``>= threshold``)."""
     return no_speech_prob >= threshold

tests/test_bridge_stt_filter.py

@@ -7,15 +7,39 @@ produces no transcript and no reply. Thresholds are config-driven, so the tests
 pass explicit references rather than hardcoding the production defaults.
 """
 
+import numpy as np
 import pytest
 
 from bridge.stt_filter import (
     filter_speech_segments,
+    has_speech,
     is_non_speech,
     segment_confidence,
 )
 
 
+@pytest.mark.unit
+def test_has_speech_rejects_silence():
+    # Pure digital silence is unambiguously non-speech: VAD must skip STT.
+    silence = np.zeros(16000, dtype=np.float32)  # 1s @ 16kHz
+    assert has_speech(silence, 16000) is False
+
+
+@pytest.mark.unit
+def test_has_speech_rejects_a_brief_loud_blip():
+    # A short loud transient (a clap / pop): mostly silence with a 50ms spike,
+    # below the min-speech duration, so no real speech region is found.
+    audio = np.zeros(16000, dtype=np.float32)
+    audio[8000:8800] = 0.9  # ~50ms full-scale burst
+    assert has_speech(audio, 16000, min_speech_duration_ms=200) is False
+
+
+@pytest.mark.unit
+def test_has_speech_fails_open_on_empty_when_vad_present_returns_false():
+    # Empty audio has nothing to transcribe; treat as non-speech.
+    assert has_speech(np.zeros(0, dtype=np.float32), 16000) is False
+
+
 class Seg:
     """Minimal stand-in for a faster-whisper segment."""