fix(bridge): gate STT on real speech so noise doesn't trigger replies
The bridge transcribe path joined every Whisper segment unconditionally, so a
brief loud sound or background noise that momentarily opened the mic gate (no
real speech) still produced a transcript, and Whisper's noise hallucinations
("감사합니다", "MBC 뉴스", ...) made the bot reply to nothing.
Add bridge/stt_filter.py mirroring the desktop listener's _filter_noisy_segments
policy: a hard no_speech_prob cutoff (whisper_no_speech_threshold) plus an
avg_logprob confidence floor (whisper_min_confidence), both config-driven. Apply
it in transcribe() so only segments that look like human speech survive; a
noise-only turn yields an empty transcript and the existing empty-transcript
guard drops it with no reply. Add unit tests for the gate.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -51,8 +51,10 @@ from flask import Flask, request, jsonify, Response, stream_with_context
|
||||
|
||||
try: # package-relative when imported as ``bridge.server``
|
||||
from bridge.text_utils import split_sentences
|
||||
from bridge.stt_filter import filter_speech_segments
|
||||
except ImportError: # script-relative when run as ``bridge/server.py``
|
||||
from text_utils import split_sentences
|
||||
from stt_filter import filter_speech_segments
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
@@ -183,7 +185,19 @@ def transcribe(wav_bytes: bytes) -> dict:
|
||||
x_new = np.linspace(0.0, 1.0, num=n_out, endpoint=False)
|
||||
audio = np.interp(x_new, x_old, audio).astype(np.float32)
|
||||
segments, info = _whisper.transcribe(audio, beam_size=1)
|
||||
text = "".join(seg.text for seg in segments).strip()
|
||||
# Speech gate: drop non-speech / hallucinated segments so a brief loud sound
|
||||
# or background noise (mic blip with no real speech) does not become a
|
||||
# transcript and make the bot reply to nothing. Mirrors the desktop
|
||||
# listener's policy, driven by the same config thresholds.
|
||||
no_speech_threshold = getattr(_cfg, "whisper_no_speech_threshold", 0.5)
|
||||
min_confidence = getattr(_cfg, "whisper_min_confidence", 0.3)
|
||||
kept = filter_speech_segments(
|
||||
segments,
|
||||
no_speech_threshold=no_speech_threshold,
|
||||
min_confidence=min_confidence,
|
||||
log=lambda m: print(f"[bridge] {m}", flush=True),
|
||||
)
|
||||
text = "".join(seg.text for seg in kept).strip()
|
||||
return {"text": text, "language": getattr(info, "language", None)}
|
||||
|
||||
|
||||
|
||||
79
bridge/stt_filter.py
Normal file
79
bridge/stt_filter.py
Normal file
@@ -0,0 +1,79 @@
|
||||
"""Speech gate for the Discord STT path.
|
||||
|
||||
Whisper will transcribe, and frequently *hallucinate*, on non-speech audio:
|
||||
silence, background noise, or a brief loud blip (a cough, a key clack, a mic
|
||||
pop) that momentarily opens the voice gate without anyone actually speaking.
|
||||
Left unfiltered those produce phantom transcripts ("MBC 뉴스", "감사합니다", ...)
|
||||
and the assistant ends up replying to noise.
|
||||
|
||||
This mirrors the desktop listener's ``_filter_noisy_segments`` policy
|
||||
(``src/jarvis/listening/listener.py``) so both entry points apply identical
|
||||
rules, both driven by the same config thresholds:
|
||||
|
||||
1. Hard ``no_speech_prob`` cutoff (``whisper_no_speech_threshold``): Whisper's
|
||||
own "this segment is not speech" probability. Checked first and
|
||||
independently of confidence, because Whisper can be *confident* about a
|
||||
hallucinated phrase on pure noise.
|
||||
2. ``avg_logprob`` confidence floor (``whisper_min_confidence``): drops
|
||||
low-quality decodes that survive the no-speech check.
|
||||
|
||||
A segment must pass both to count as real human speech.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Callable, Optional
|
||||
|
||||
|
||||
def is_non_speech(no_speech_prob: float, threshold: float) -> bool:
|
||||
"""True when Whisper flags a segment as non-speech (``>= threshold``)."""
|
||||
return no_speech_prob >= threshold
|
||||
|
||||
|
||||
def segment_confidence(seg) -> Optional[float]:
|
||||
"""Map a Whisper segment to a 0..1 confidence.
|
||||
|
||||
Prefers ``avg_logprob`` (mapped to 0..1 the same way the desktop listener
|
||||
does), falling back to ``1 - no_speech_prob`` when the log-prob is absent.
|
||||
Returns ``None`` when neither signal is available so the caller keeps the
|
||||
segment rather than dropping it on missing metadata.
|
||||
"""
|
||||
avg = getattr(seg, "avg_logprob", None)
|
||||
if avg is not None:
|
||||
return min(1.0, max(0.0, avg + 1.0))
|
||||
nsp = getattr(seg, "no_speech_prob", None)
|
||||
if nsp is not None:
|
||||
return 1.0 - nsp
|
||||
return None
|
||||
|
||||
|
||||
def filter_speech_segments(
|
||||
segments,
|
||||
*,
|
||||
no_speech_threshold: float = 0.5,
|
||||
min_confidence: float = 0.3,
|
||||
log: Optional[Callable[[str], None]] = None,
|
||||
) -> list:
|
||||
"""Keep only the segments that look like real human speech, in order.
|
||||
|
||||
``log(msg)``, if given, is called with a short reason for each dropped
|
||||
segment (used by the bridge to surface why a noisy turn produced no reply).
|
||||
"""
|
||||
kept = []
|
||||
for seg in segments:
|
||||
nsp = getattr(seg, "no_speech_prob", None)
|
||||
if nsp is not None and is_non_speech(nsp, no_speech_threshold):
|
||||
if log:
|
||||
log(f"segment dropped (no_speech_prob={nsp:.2f}): {_preview(seg)}")
|
||||
continue
|
||||
conf = segment_confidence(seg)
|
||||
if conf is not None and conf < min_confidence:
|
||||
if log:
|
||||
log(f"segment dropped (confidence={conf:.2f}): {_preview(seg)}")
|
||||
continue
|
||||
kept.append(seg)
|
||||
return kept
|
||||
|
||||
|
||||
def _preview(seg) -> str:
|
||||
return repr(getattr(seg, "text", "").strip()[:50])
|
||||
Reference in New Issue
Block a user