feat(bridge): gate Whisper behind Silero VAD; harden broadcast auto-start
Address review of the noise/broadcast fixes: - STT now refuses to run Whisper on non-speech. transcribe() runs the Silero VAD (bundled with faster-whisper, no new dep) BEFORE the model, so noise or a brief loud blip with no real speech never reaches STT and can't be hallucinated into a transcript. The no_speech_prob/avg_logprob post-filter stays as a second line of defence (a clap the VAD lets through is still killed by Whisper's own no_speech_prob). VAD is env-tunable (VAD_THRESHOLD, VAD_MIN_SPEECH_MS, VAD_ENABLED) and fail-open so a VAD error never swallows a real utterance. Validated on real audio: synthesised Korean speech passes; silence, a 50ms blip and white noise are rejected. - Broadcast auto-start no longer blocks the voice join and no longer silently swallows failures: wiring is synchronous, the Go-Live start runs in the background with a bounded retry and a loud final-failure log. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -51,10 +51,10 @@ from flask import Flask, request, jsonify, Response, stream_with_context
|
||||
|
||||
try: # package-relative when imported as ``bridge.server``
|
||||
from bridge.text_utils import split_sentences
|
||||
from bridge.stt_filter import filter_speech_segments
|
||||
from bridge.stt_filter import filter_speech_segments, has_speech
|
||||
except ImportError: # script-relative when run as ``bridge/server.py``
|
||||
from text_utils import split_sentences
|
||||
from stt_filter import filter_speech_segments
|
||||
from stt_filter import filter_speech_segments, has_speech
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
@@ -66,6 +66,15 @@ BRIDGE_PORT = int(os.environ.get("BRIDGE_PORT", "8765"))
|
||||
BRAIN_ENABLED = os.environ.get("JARVIS_BRAIN_ENABLED", "1") not in ("0", "false", "False")
|
||||
TTS_ENABLED = os.environ.get("JARVIS_TTS_ENABLED", "1") not in ("0", "false", "False")
|
||||
|
||||
# Pre-STT speech gate (Silero VAD). Tunable for the Discord mic without a code
|
||||
# change: raise VAD_THRESHOLD to reject more noise, lower it to catch quieter
|
||||
# speech. VAD_MIN_SPEECH_MS is the shortest run of speech that counts (a brief
|
||||
# loud blip shorter than this never reaches Whisper). Set VAD_ENABLED=0 to fall
|
||||
# back to the old behaviour (always transcribe, rely on the post-filter only).
|
||||
VAD_ENABLED = os.environ.get("VAD_ENABLED", "1") not in ("0", "false", "False")
|
||||
VAD_THRESHOLD = float(os.environ.get("VAD_THRESHOLD", "0.4"))
|
||||
VAD_MIN_SPEECH_MS = int(os.environ.get("VAD_MIN_SPEECH_MS", "200"))
|
||||
|
||||
# TTS engine: "melo" (MeloTTS Korean speaker, the warm worker) is the primary
|
||||
# voice; Piper is kept as a fallback if the worker is unreachable. Set
|
||||
# TTS_ENGINE=piper to disable MeloTTS entirely.
|
||||
@@ -184,10 +193,25 @@ def transcribe(wav_bytes: bytes) -> dict:
|
||||
x_old = np.linspace(0.0, 1.0, num=audio.size, endpoint=False)
|
||||
x_new = np.linspace(0.0, 1.0, num=n_out, endpoint=False)
|
||||
audio = np.interp(x_new, x_old, audio).astype(np.float32)
|
||||
# Pre-STT speech gate: don't even invoke Whisper unless there is real speech
|
||||
# in the clip. Noise or a brief loud blip (no actual speech) is dropped here,
|
||||
# before transcription, so the model never gets a chance to hallucinate a
|
||||
# phrase from it. Fail-open inside has_speech() keeps a real utterance from
|
||||
# being swallowed if the VAD is unavailable.
|
||||
if VAD_ENABLED and not has_speech(
|
||||
audio,
|
||||
16000,
|
||||
threshold=VAD_THRESHOLD,
|
||||
min_speech_duration_ms=VAD_MIN_SPEECH_MS,
|
||||
log=lambda m: print(f"[bridge] {m}", flush=True),
|
||||
):
|
||||
print("[bridge] no speech detected (VAD) — skipping STT", flush=True)
|
||||
return {"text": "", "language": None}
|
||||
|
||||
segments, info = _whisper.transcribe(audio, beam_size=1)
|
||||
# Speech gate: drop non-speech / hallucinated segments so a brief loud sound
|
||||
# or background noise (mic blip with no real speech) does not become a
|
||||
# transcript and make the bot reply to nothing. Mirrors the desktop
|
||||
# Second line of defence: even speech-like audio (e.g. a clap the VAD let
|
||||
# through) can make Whisper hallucinate, so drop non-speech / low-confidence
|
||||
# segments by Whisper's own no_speech_prob + avg_logprob. Mirrors the desktop
|
||||
# listener's policy, driven by the same config thresholds.
|
||||
no_speech_threshold = getattr(_cfg, "whisper_no_speech_threshold", 0.5)
|
||||
min_confidence = getattr(_cfg, "whisper_min_confidence", 0.3)
|
||||
|
||||
Reference in New Issue
Block a user