feat(bridge): gate Whisper behind Silero VAD; harden broadcast auto-start

Address review of the noise/broadcast fixes:

- STT now refuses to run Whisper on non-speech. transcribe() runs the Silero
  VAD (bundled with faster-whisper, no new dep) BEFORE the model, so noise or a
  brief loud blip with no real speech never reaches STT and can't be
  hallucinated into a transcript. The no_speech_prob/avg_logprob post-filter
  stays as a second line of defence (a clap the VAD lets through is still killed
  by Whisper's own no_speech_prob). VAD is env-tunable (VAD_THRESHOLD,
  VAD_MIN_SPEECH_MS, VAD_ENABLED) and fail-open so a VAD error never swallows a
  real utterance. Validated on real audio: synthesised Korean speech passes;
  silence, a 50ms blip and white noise are rejected.

- Broadcast auto-start no longer blocks the voice join and no longer silently
  swallows failures: wiring is synchronous, the Go-Live start runs in the
  background with a bounded retry and a loud final-failure log.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-06-13 14:53:54 +09:00
parent 39c7a22a12
commit 6d72e10f9c
5 changed files with 194 additions and 48 deletions

View File

@@ -51,10 +51,10 @@ from flask import Flask, request, jsonify, Response, stream_with_context
try: # package-relative when imported as ``bridge.server``
from bridge.text_utils import split_sentences
from bridge.stt_filter import filter_speech_segments
from bridge.stt_filter import filter_speech_segments, has_speech
except ImportError: # script-relative when run as ``bridge/server.py``
from text_utils import split_sentences
from stt_filter import filter_speech_segments
from stt_filter import filter_speech_segments, has_speech
app = Flask(__name__)
@@ -66,6 +66,15 @@ BRIDGE_PORT = int(os.environ.get("BRIDGE_PORT", "8765"))
BRAIN_ENABLED = os.environ.get("JARVIS_BRAIN_ENABLED", "1") not in ("0", "false", "False")
TTS_ENABLED = os.environ.get("JARVIS_TTS_ENABLED", "1") not in ("0", "false", "False")
# Pre-STT speech gate (Silero VAD). Tunable for the Discord mic without a code
# change: raise VAD_THRESHOLD to reject more noise, lower it to catch quieter
# speech. VAD_MIN_SPEECH_MS is the shortest run of speech that counts (a brief
# loud blip shorter than this never reaches Whisper). Set VAD_ENABLED=0 to fall
# back to the old behaviour (always transcribe, rely on the post-filter only).
VAD_ENABLED = os.environ.get("VAD_ENABLED", "1") not in ("0", "false", "False")
VAD_THRESHOLD = float(os.environ.get("VAD_THRESHOLD", "0.4"))
VAD_MIN_SPEECH_MS = int(os.environ.get("VAD_MIN_SPEECH_MS", "200"))
# TTS engine: "melo" (MeloTTS Korean speaker, the warm worker) is the primary
# voice; Piper is kept as a fallback if the worker is unreachable. Set
# TTS_ENGINE=piper to disable MeloTTS entirely.
@@ -184,10 +193,25 @@ def transcribe(wav_bytes: bytes) -> dict:
x_old = np.linspace(0.0, 1.0, num=audio.size, endpoint=False)
x_new = np.linspace(0.0, 1.0, num=n_out, endpoint=False)
audio = np.interp(x_new, x_old, audio).astype(np.float32)
# Pre-STT speech gate: don't even invoke Whisper unless there is real speech
# in the clip. Noise or a brief loud blip (no actual speech) is dropped here,
# before transcription, so the model never gets a chance to hallucinate a
# phrase from it. Fail-open inside has_speech() keeps a real utterance from
# being swallowed if the VAD is unavailable.
if VAD_ENABLED and not has_speech(
audio,
16000,
threshold=VAD_THRESHOLD,
min_speech_duration_ms=VAD_MIN_SPEECH_MS,
log=lambda m: print(f"[bridge] {m}", flush=True),
):
print("[bridge] no speech detected (VAD) — skipping STT", flush=True)
return {"text": "", "language": None}
segments, info = _whisper.transcribe(audio, beam_size=1)
# Speech gate: drop non-speech / hallucinated segments so a brief loud sound
# or background noise (mic blip with no real speech) does not become a
# transcript and make the bot reply to nothing. Mirrors the desktop
# Second line of defence: even speech-like audio (e.g. a clap the VAD let
# through) can make Whisper hallucinate, so drop non-speech / low-confidence
# segments by Whisper's own no_speech_prob + avg_logprob. Mirrors the desktop
# listener's policy, driven by the same config thresholds.
no_speech_threshold = getattr(_cfg, "whisper_no_speech_threshold", 0.5)
min_confidence = getattr(_cfg, "whisper_min_confidence", 0.3)