fix(bridge): gate STT on real speech so noise doesn't trigger replies

The bridge transcribe path joined every Whisper segment unconditionally, so a
brief loud sound or background noise that momentarily opened the mic gate (no
real speech) still produced a transcript, and Whisper's noise hallucinations
("감사합니다", "MBC 뉴스", ...) made the bot reply to nothing.

Add bridge/stt_filter.py mirroring the desktop listener's _filter_noisy_segments
policy: a hard no_speech_prob cutoff (whisper_no_speech_threshold) plus an
avg_logprob confidence floor (whisper_min_confidence), both config-driven. Apply
it in transcribe() so only segments that look like human speech survive; a
noise-only turn yields an empty transcript and the existing empty-transcript
guard drops it with no reply. Add unit tests for the gate.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-06-13 14:45:22 +09:00
parent 568a1ae50b
commit 39c7a22a12
3 changed files with 196 additions and 1 deletions

View File

@@ -51,8 +51,10 @@ from flask import Flask, request, jsonify, Response, stream_with_context
try: # package-relative when imported as ``bridge.server``
from bridge.text_utils import split_sentences
from bridge.stt_filter import filter_speech_segments
except ImportError: # script-relative when run as ``bridge/server.py``
from text_utils import split_sentences
from stt_filter import filter_speech_segments
app = Flask(__name__)
@@ -183,7 +185,19 @@ def transcribe(wav_bytes: bytes) -> dict:
x_new = np.linspace(0.0, 1.0, num=n_out, endpoint=False)
audio = np.interp(x_new, x_old, audio).astype(np.float32)
segments, info = _whisper.transcribe(audio, beam_size=1)
text = "".join(seg.text for seg in segments).strip()
# Speech gate: drop non-speech / hallucinated segments so a brief loud sound
# or background noise (mic blip with no real speech) does not become a
# transcript and make the bot reply to nothing. Mirrors the desktop
# listener's policy, driven by the same config thresholds.
no_speech_threshold = getattr(_cfg, "whisper_no_speech_threshold", 0.5)
min_confidence = getattr(_cfg, "whisper_min_confidence", 0.3)
kept = filter_speech_segments(
segments,
no_speech_threshold=no_speech_threshold,
min_confidence=min_confidence,
log=lambda m: print(f"[bridge] {m}", flush=True),
)
text = "".join(seg.text for seg in kept).strip()
return {"text": text, "language": getattr(info, "language", None)}

79
bridge/stt_filter.py Normal file
View File

@@ -0,0 +1,79 @@
"""Speech gate for the Discord STT path.
Whisper will transcribe, and frequently *hallucinate*, on non-speech audio:
silence, background noise, or a brief loud blip (a cough, a key clack, a mic
pop) that momentarily opens the voice gate without anyone actually speaking.
Left unfiltered those produce phantom transcripts ("MBC 뉴스", "감사합니다", ...)
and the assistant ends up replying to noise.
This mirrors the desktop listener's ``_filter_noisy_segments`` policy
(``src/jarvis/listening/listener.py``) so both entry points apply identical
rules, both driven by the same config thresholds:
1. Hard ``no_speech_prob`` cutoff (``whisper_no_speech_threshold``): Whisper's
own "this segment is not speech" probability. Checked first and
independently of confidence, because Whisper can be *confident* about a
hallucinated phrase on pure noise.
2. ``avg_logprob`` confidence floor (``whisper_min_confidence``): drops
low-quality decodes that survive the no-speech check.
A segment must pass both to count as real human speech.
"""
from __future__ import annotations
from typing import Callable, Optional
def is_non_speech(no_speech_prob: float, threshold: float) -> bool:
"""True when Whisper flags a segment as non-speech (``>= threshold``)."""
return no_speech_prob >= threshold
def segment_confidence(seg) -> Optional[float]:
"""Map a Whisper segment to a 0..1 confidence.
Prefers ``avg_logprob`` (mapped to 0..1 the same way the desktop listener
does), falling back to ``1 - no_speech_prob`` when the log-prob is absent.
Returns ``None`` when neither signal is available so the caller keeps the
segment rather than dropping it on missing metadata.
"""
avg = getattr(seg, "avg_logprob", None)
if avg is not None:
return min(1.0, max(0.0, avg + 1.0))
nsp = getattr(seg, "no_speech_prob", None)
if nsp is not None:
return 1.0 - nsp
return None
def filter_speech_segments(
segments,
*,
no_speech_threshold: float = 0.5,
min_confidence: float = 0.3,
log: Optional[Callable[[str], None]] = None,
) -> list:
"""Keep only the segments that look like real human speech, in order.
``log(msg)``, if given, is called with a short reason for each dropped
segment (used by the bridge to surface why a noisy turn produced no reply).
"""
kept = []
for seg in segments:
nsp = getattr(seg, "no_speech_prob", None)
if nsp is not None and is_non_speech(nsp, no_speech_threshold):
if log:
log(f"segment dropped (no_speech_prob={nsp:.2f}): {_preview(seg)}")
continue
conf = segment_confidence(seg)
if conf is not None and conf < min_confidence:
if log:
log(f"segment dropped (confidence={conf:.2f}): {_preview(seg)}")
continue
kept.append(seg)
return kept
def _preview(seg) -> str:
return repr(getattr(seg, "text", "").strip()[:50])