Compare commits
3 Commits
d6c029d7d5
...
6d72e10f9c
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6d72e10f9c | ||
|
|
39c7a22a12 | ||
|
|
568a1ae50b |
93
bot/src/broadcast.test.ts
Normal file
93
bot/src/broadcast.test.ts
Normal file
@@ -0,0 +1,93 @@
|
||||
import { test, expect } from "bun:test";
|
||||
|
||||
// broadcast.ts pulls in the runtime `config`, which requires DISCORD_GUILD_ID.
|
||||
// Set it before the dynamic import so the module loads without a real .env.
|
||||
process.env.DISCORD_GUILD_ID ||= "test-guild";
|
||||
const { wireBroadcast, autoStartBroadcast } = await import("./broadcast.ts");
|
||||
|
||||
/** Minimal in-memory ScreenStreamer that records start/stop calls. */
|
||||
function fakeStreamer(active = false) {
|
||||
const calls: string[] = [];
|
||||
let on = active;
|
||||
return {
|
||||
calls,
|
||||
kind: "selfbot" as const,
|
||||
isActive: () => on,
|
||||
async start() {
|
||||
calls.push("start");
|
||||
on = true;
|
||||
return "ok";
|
||||
},
|
||||
async stop() {
|
||||
calls.push("stop");
|
||||
on = false;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const ctx = { guildId: "g", voiceChannelId: "c" };
|
||||
|
||||
test("wireBroadcast reports live state to the brain", () => {
|
||||
const s = fakeStreamer(false);
|
||||
const session: any = {};
|
||||
wireBroadcast(session, s as any, ctx);
|
||||
expect(session.getBroadcasting()).toBe(false);
|
||||
});
|
||||
|
||||
test("wireBroadcast lets the brain toggle the stream by voice", async () => {
|
||||
const s = fakeStreamer(false);
|
||||
const session: any = {};
|
||||
wireBroadcast(session, s as any, ctx);
|
||||
await session.onBroadcastAction("start"); // idle -> start
|
||||
expect(session.getBroadcasting()).toBe(true);
|
||||
await session.onBroadcastAction("start"); // already live -> no-op
|
||||
await session.onBroadcastAction("stop"); // stop
|
||||
expect(session.getBroadcasting()).toBe(false);
|
||||
expect(s.calls).toEqual(["start", "stop"]);
|
||||
});
|
||||
|
||||
test("autoStartBroadcast starts the broadcast when idle", async () => {
|
||||
const s = fakeStreamer(false);
|
||||
const ok = await autoStartBroadcast(s as any, ctx, 2, 0);
|
||||
expect(ok).toBe(true);
|
||||
expect(s.calls).toEqual(["start"]);
|
||||
});
|
||||
|
||||
test("autoStartBroadcast does not restart an already-live broadcast", async () => {
|
||||
const s = fakeStreamer(true);
|
||||
const ok = await autoStartBroadcast(s as any, ctx, 2, 0);
|
||||
expect(ok).toBe(true);
|
||||
expect(s.calls).toEqual([]);
|
||||
});
|
||||
|
||||
test("autoStartBroadcast retries a transient failure and then succeeds", async () => {
|
||||
let on = false;
|
||||
const calls: string[] = [];
|
||||
const s: any = {
|
||||
isActive: () => on,
|
||||
async start() {
|
||||
calls.push("start");
|
||||
if (calls.length === 1) throw new Error("transient: voice not ready");
|
||||
on = true;
|
||||
},
|
||||
async stop() {},
|
||||
};
|
||||
const ok = await autoStartBroadcast(s, ctx, 2, 0);
|
||||
expect(ok).toBe(true);
|
||||
expect(calls).toEqual(["start", "start"]);
|
||||
});
|
||||
|
||||
test("autoStartBroadcast returns false after exhausting retries (no throw)", async () => {
|
||||
let attempts = 0;
|
||||
const s: any = {
|
||||
isActive: () => false,
|
||||
async start() {
|
||||
attempts++;
|
||||
throw new Error("boom");
|
||||
},
|
||||
async stop() {},
|
||||
};
|
||||
const ok = await autoStartBroadcast(s, ctx, 2, 0);
|
||||
expect(ok).toBe(false);
|
||||
expect(attempts).toBe(2); // retried, then gave up without throwing
|
||||
});
|
||||
121
bot/src/broadcast.ts
Normal file
121
bot/src/broadcast.ts
Normal file
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* Broadcast <-> voice coupling, shared by the normal-bot and userbot entry
|
||||
* points so both modes behave identically.
|
||||
*
|
||||
* The screen broadcast is coupled to the voice session: it auto-starts when the
|
||||
* assistant joins a voice channel (screen-share mode default), reports its live
|
||||
* state to the brain each turn (search routes Chrome while live / Gemini when
|
||||
* off), and the brain can toggle it by voice ("방송 켜줘 / 꺼줘"). This logic
|
||||
* used to live only in the normal-bot join handler, so the userbot path (the
|
||||
* only mode that can actually Go Live) never started a broadcast. Keeping it
|
||||
* here means a single implementation serves both.
|
||||
*/
|
||||
import { config } from "./config.ts";
|
||||
import { createStreamer, type ScreenStreamer, type StreamContext } from "./stream/index.ts";
|
||||
import type { VoiceSession } from "./voice.ts";
|
||||
|
||||
/** One streamer per guild, shared across both entry points. */
|
||||
const streamers = new Map<string, ScreenStreamer>();
|
||||
|
||||
export async function getStreamer(guildId: string): Promise<ScreenStreamer> {
|
||||
let s = streamers.get(guildId);
|
||||
if (!s) {
|
||||
s = await createStreamer(config);
|
||||
streamers.set(guildId, s);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
/** The existing streamer for a guild, if one has been created. */
|
||||
export function peekStreamer(guildId: string): ScreenStreamer | undefined {
|
||||
return streamers.get(guildId);
|
||||
}
|
||||
|
||||
type BroadcastSession = Pick<VoiceSession, "getBroadcasting" | "onBroadcastAction">;
|
||||
|
||||
/**
|
||||
* Wire a voice session to a streamer (no auto-start). Pure (no config / registry
|
||||
* access) so it can be unit-tested with a fake streamer:
|
||||
* - report live state to the brain each turn,
|
||||
* - let the brain toggle the stream by voice.
|
||||
*/
|
||||
export function wireBroadcast(
|
||||
session: BroadcastSession,
|
||||
streamer: ScreenStreamer,
|
||||
ctx: StreamContext,
|
||||
): void {
|
||||
session.getBroadcasting = () => streamer.isActive();
|
||||
session.onBroadcastAction = async (action) => {
|
||||
if (action === "start") {
|
||||
if (!streamer.isActive()) await streamer.start(ctx);
|
||||
} else if (streamer.isActive()) {
|
||||
await streamer.stop();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-start the broadcast, retrying a couple of times before giving up. The
|
||||
* selfbot Go-Live path takes ~10-15s of humanised delays and can fail on a
|
||||
* transient (login race, voice not ready yet), so a bounded retry recovers the
|
||||
* common case. On final failure it logs loudly rather than silently leaving the
|
||||
* user in voice with no broadcast. Never throws: a broadcast problem must not
|
||||
* tear down the voice conversation. Returns whether the stream ended up live.
|
||||
*/
|
||||
export async function autoStartBroadcast(
|
||||
streamer: ScreenStreamer,
|
||||
ctx: StreamContext,
|
||||
attempts = 2,
|
||||
retryDelayMs = 1500,
|
||||
): Promise<boolean> {
|
||||
for (let attempt = 1; attempt <= attempts; attempt++) {
|
||||
try {
|
||||
if (streamer.isActive()) return true;
|
||||
await streamer.start(ctx);
|
||||
if (streamer.isActive()) return true;
|
||||
console.error(
|
||||
`[broadcast] auto-start attempt ${attempt}/${attempts} did not go live`,
|
||||
);
|
||||
} catch (e) {
|
||||
console.error(`[broadcast] auto-start attempt ${attempt}/${attempts} failed:`, e);
|
||||
}
|
||||
if (attempt < attempts && retryDelayMs > 0) {
|
||||
await new Promise((r) => setTimeout(r, retryDelayMs));
|
||||
}
|
||||
}
|
||||
console.error(
|
||||
"[broadcast] ⚠️ auto-start FAILED after retries — voice is up but NOT broadcasting " +
|
||||
"(STREAM_BROWSER=true). Trigger it by voice ('방송 켜줘') or check the selfbot stream deps.",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Couple the broadcast to a freshly joined voice session when screen-share mode
|
||||
* is on (STREAM_BROWSER=true). No-op in voice-only mode (STREAM_BROWSER=false).
|
||||
*
|
||||
* The toggle hooks are wired synchronously; the auto-start runs in the
|
||||
* BACKGROUND so the ~10-15s Go-Live handshake never blocks the voice session
|
||||
* from coming up. The bot can already hear and reply while the stream warms.
|
||||
*/
|
||||
export async function maybeCoupleBroadcast(
|
||||
session: BroadcastSession,
|
||||
ctx: StreamContext,
|
||||
): Promise<void> {
|
||||
if (!config.screenBrowser) return;
|
||||
const streamer = await getStreamer(ctx.guildId);
|
||||
wireBroadcast(session, streamer, ctx);
|
||||
void autoStartBroadcast(streamer, ctx);
|
||||
}
|
||||
|
||||
/** Stop the broadcast for a guild if one is live (e.g. on leave). */
|
||||
export async function stopBroadcast(guildId: string): Promise<void> {
|
||||
const s = streamers.get(guildId);
|
||||
if (s?.isActive()) {
|
||||
try {
|
||||
await s.stop();
|
||||
} catch (e) {
|
||||
console.error("[broadcast] stop failed:", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -19,23 +19,13 @@ import { AttachmentBuilder } from "discord.js";
|
||||
import { config } from "./config.ts";
|
||||
import { ask, health } from "./bridge.ts";
|
||||
import { joinChannel, leaveGuild, getSession } from "./voice.ts";
|
||||
import { createStreamer, type ScreenStreamer, type StreamContext } from "./stream/index.ts";
|
||||
import { type StreamContext } from "./stream/index.ts";
|
||||
import { getStreamer, peekStreamer, maybeCoupleBroadcast, stopBroadcast } from "./broadcast.ts";
|
||||
|
||||
const client = new Client({
|
||||
intents: [GatewayIntentBits.Guilds, GatewayIntentBits.GuildVoiceStates],
|
||||
});
|
||||
|
||||
const streamers = new Map<string, ScreenStreamer>();
|
||||
|
||||
async function getStreamer(guildId: string): Promise<ScreenStreamer> {
|
||||
let s = streamers.get(guildId);
|
||||
if (!s) {
|
||||
s = await createStreamer(config);
|
||||
streamers.set(guildId, s);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
const eph = { flags: MessageFlags.Ephemeral } as const;
|
||||
|
||||
client.once("clientReady", () => {
|
||||
@@ -87,23 +77,7 @@ async function handleJoin(i: ChatInputCommandInteraction) {
|
||||
// each turn (search routes Chrome while live / Gemini when off), and let the
|
||||
// brain toggle it via voice ("방송 켜줘 / 꺼줘"). In voice-only mode
|
||||
// (STREAM_BROWSER=false) none of this runs and the broadcast stays off.
|
||||
if (config.screenBrowser) {
|
||||
const streamer = await getStreamer(i.guildId!);
|
||||
const ctx: StreamContext = { guildId: i.guildId!, voiceChannelId: channel.id };
|
||||
session.getBroadcasting = () => streamer.isActive();
|
||||
session.onBroadcastAction = async (action) => {
|
||||
if (action === "start") {
|
||||
if (!streamer.isActive()) await streamer.start(ctx);
|
||||
} else if (streamer.isActive()) {
|
||||
await streamer.stop();
|
||||
}
|
||||
};
|
||||
try {
|
||||
if (!streamer.isActive()) await streamer.start(ctx);
|
||||
} catch (e) {
|
||||
console.error("[join] auto-broadcast failed:", e);
|
||||
}
|
||||
}
|
||||
await maybeCoupleBroadcast(session, { guildId: i.guildId!, voiceChannelId: channel.id });
|
||||
|
||||
await i.editReply(`🎙️ '${channel.name}' 채널에 접속했습니다. 말씀하세요.`);
|
||||
}
|
||||
@@ -112,14 +86,7 @@ async function handleLeave(i: ChatInputCommandInteraction) {
|
||||
const left = leaveGuild(i.guildId!);
|
||||
// Leaving voice also ends the broadcast — don't leave a stream live with no
|
||||
// session driving it.
|
||||
const streamer = streamers.get(i.guildId!);
|
||||
if (streamer?.isActive()) {
|
||||
try {
|
||||
await streamer.stop();
|
||||
} catch (e) {
|
||||
console.error("[leave] stopping broadcast failed:", e);
|
||||
}
|
||||
}
|
||||
await stopBroadcast(i.guildId!);
|
||||
await i.reply({ content: left ? "음성 채널에서 나갔습니다." : "접속 중인 세션이 없습니다.", ...eph });
|
||||
}
|
||||
|
||||
@@ -158,7 +125,7 @@ async function handleStream(i: ChatInputCommandInteraction) {
|
||||
}
|
||||
|
||||
async function handleStop(i: ChatInputCommandInteraction) {
|
||||
const streamer = streamers.get(i.guildId!);
|
||||
const streamer = peekStreamer(i.guildId!);
|
||||
if (!streamer) return i.reply({ content: "송출 중이 아닙니다.", ...eph });
|
||||
await streamer.stop();
|
||||
await i.reply({ content: "송출을 중단했습니다.", ...eph });
|
||||
@@ -174,7 +141,7 @@ async function handleStatus(i: ChatInputCommandInteraction) {
|
||||
/* keep unreachable */
|
||||
}
|
||||
const session = getSession(i.guildId!);
|
||||
const streamer = streamers.get(i.guildId!);
|
||||
const streamer = peekStreamer(i.guildId!);
|
||||
await i.editReply(
|
||||
[
|
||||
`브릿지 두뇌: ${brain}`,
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
import type { VoiceBasedChannel } from "discord.js";
|
||||
import { config } from "./config.ts";
|
||||
import { joinChannel, leaveGuild } from "./voice.ts";
|
||||
import { maybeCoupleBroadcast, stopBroadcast } from "./broadcast.ts";
|
||||
|
||||
type AnyClient = any;
|
||||
|
||||
@@ -43,6 +44,11 @@ async function joinAndListen(client: AnyClient, channelId: string): Promise<void
|
||||
// joinVoiceChannel (it exposes id, guild.id and guild.voiceAdapterCreator).
|
||||
const session = await joinChannel(channel as unknown as VoiceBasedChannel);
|
||||
session.onTurn = ({ transcript, reply }) => console.log(`🗣️ ${transcript}\n🤖 ${reply}`);
|
||||
// Screen-share mode (STREAM_BROWSER=true): auto-start the broadcast on join,
|
||||
// report its live state to the brain each turn, and let the brain toggle it by
|
||||
// voice. Userbot is the only mode that can actually Go Live, so without this
|
||||
// wiring the broadcast never starts. No-op when STREAM_BROWSER=false.
|
||||
await maybeCoupleBroadcast(session, { guildId: channel.guild.id, voiceChannelId: channel.id });
|
||||
console.log(`🎙️ 유저봇이 '${channel.name}' 음성채널에 참여했습니다.`);
|
||||
}
|
||||
|
||||
@@ -72,6 +78,9 @@ export async function runUserbot(): Promise<void> {
|
||||
const ch = msg.member?.voice?.channel || msg.author?.voice?.channel;
|
||||
if (ch) await joinAndListen(client, ch.id).catch((e) => console.error("[userbot] join cmd:", e));
|
||||
} else if (content === "!자비스 leave" || content === "!jarvis leave") {
|
||||
// Leaving voice also ends the broadcast — don't leave a stream live with
|
||||
// no session driving it.
|
||||
await stopBroadcast(config.guildId);
|
||||
leaveGuild(config.guildId);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -51,8 +51,10 @@ from flask import Flask, request, jsonify, Response, stream_with_context
|
||||
|
||||
try: # package-relative when imported as ``bridge.server``
|
||||
from bridge.text_utils import split_sentences
|
||||
from bridge.stt_filter import filter_speech_segments, has_speech
|
||||
except ImportError: # script-relative when run as ``bridge/server.py``
|
||||
from text_utils import split_sentences
|
||||
from stt_filter import filter_speech_segments, has_speech
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
@@ -64,6 +66,15 @@ BRIDGE_PORT = int(os.environ.get("BRIDGE_PORT", "8765"))
|
||||
BRAIN_ENABLED = os.environ.get("JARVIS_BRAIN_ENABLED", "1") not in ("0", "false", "False")
|
||||
TTS_ENABLED = os.environ.get("JARVIS_TTS_ENABLED", "1") not in ("0", "false", "False")
|
||||
|
||||
# Pre-STT speech gate (Silero VAD). Tunable for the Discord mic without a code
|
||||
# change: raise VAD_THRESHOLD to reject more noise, lower it to catch quieter
|
||||
# speech. VAD_MIN_SPEECH_MS is the shortest run of speech that counts (a brief
|
||||
# loud blip shorter than this never reaches Whisper). Set VAD_ENABLED=0 to fall
|
||||
# back to the old behaviour (always transcribe, rely on the post-filter only).
|
||||
VAD_ENABLED = os.environ.get("VAD_ENABLED", "1") not in ("0", "false", "False")
|
||||
VAD_THRESHOLD = float(os.environ.get("VAD_THRESHOLD", "0.4"))
|
||||
VAD_MIN_SPEECH_MS = int(os.environ.get("VAD_MIN_SPEECH_MS", "200"))
|
||||
|
||||
# TTS engine: "melo" (MeloTTS Korean speaker, the warm worker) is the primary
|
||||
# voice; Piper is kept as a fallback if the worker is unreachable. Set
|
||||
# TTS_ENGINE=piper to disable MeloTTS entirely.
|
||||
@@ -182,8 +193,35 @@ def transcribe(wav_bytes: bytes) -> dict:
|
||||
x_old = np.linspace(0.0, 1.0, num=audio.size, endpoint=False)
|
||||
x_new = np.linspace(0.0, 1.0, num=n_out, endpoint=False)
|
||||
audio = np.interp(x_new, x_old, audio).astype(np.float32)
|
||||
# Pre-STT speech gate: don't even invoke Whisper unless there is real speech
|
||||
# in the clip. Noise or a brief loud blip (no actual speech) is dropped here,
|
||||
# before transcription, so the model never gets a chance to hallucinate a
|
||||
# phrase from it. Fail-open inside has_speech() keeps a real utterance from
|
||||
# being swallowed if the VAD is unavailable.
|
||||
if VAD_ENABLED and not has_speech(
|
||||
audio,
|
||||
16000,
|
||||
threshold=VAD_THRESHOLD,
|
||||
min_speech_duration_ms=VAD_MIN_SPEECH_MS,
|
||||
log=lambda m: print(f"[bridge] {m}", flush=True),
|
||||
):
|
||||
print("[bridge] no speech detected (VAD) — skipping STT", flush=True)
|
||||
return {"text": "", "language": None}
|
||||
|
||||
segments, info = _whisper.transcribe(audio, beam_size=1)
|
||||
text = "".join(seg.text for seg in segments).strip()
|
||||
# Second line of defence: even speech-like audio (e.g. a clap the VAD let
|
||||
# through) can make Whisper hallucinate, so drop non-speech / low-confidence
|
||||
# segments by Whisper's own no_speech_prob + avg_logprob. Mirrors the desktop
|
||||
# listener's policy, driven by the same config thresholds.
|
||||
no_speech_threshold = getattr(_cfg, "whisper_no_speech_threshold", 0.5)
|
||||
min_confidence = getattr(_cfg, "whisper_min_confidence", 0.3)
|
||||
kept = filter_speech_segments(
|
||||
segments,
|
||||
no_speech_threshold=no_speech_threshold,
|
||||
min_confidence=min_confidence,
|
||||
log=lambda m: print(f"[bridge] {m}", flush=True),
|
||||
)
|
||||
text = "".join(seg.text for seg in kept).strip()
|
||||
return {"text": text, "language": getattr(info, "language", None)}
|
||||
|
||||
|
||||
|
||||
122
bridge/stt_filter.py
Normal file
122
bridge/stt_filter.py
Normal file
@@ -0,0 +1,122 @@
|
||||
"""Speech gate for the Discord STT path.
|
||||
|
||||
Whisper will transcribe, and frequently *hallucinate*, on non-speech audio:
|
||||
silence, background noise, or a brief loud blip (a cough, a key clack, a mic
|
||||
pop) that momentarily opens the voice gate without anyone actually speaking.
|
||||
Left unfiltered those produce phantom transcripts ("MBC 뉴스", "감사합니다", ...)
|
||||
and the assistant ends up replying to noise.
|
||||
|
||||
This mirrors the desktop listener's ``_filter_noisy_segments`` policy
|
||||
(``src/jarvis/listening/listener.py``) so both entry points apply identical
|
||||
rules, both driven by the same config thresholds:
|
||||
|
||||
1. Hard ``no_speech_prob`` cutoff (``whisper_no_speech_threshold``): Whisper's
|
||||
own "this segment is not speech" probability. Checked first and
|
||||
independently of confidence, because Whisper can be *confident* about a
|
||||
hallucinated phrase on pure noise.
|
||||
2. ``avg_logprob`` confidence floor (``whisper_min_confidence``): drops
|
||||
low-quality decodes that survive the no-speech check.
|
||||
|
||||
A segment must pass both to count as real human speech.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Callable, Optional
|
||||
|
||||
|
||||
def has_speech(
|
||||
audio,
|
||||
sampling_rate: int = 16000,
|
||||
*,
|
||||
threshold: float = 0.4,
|
||||
min_speech_duration_ms: int = 200,
|
||||
min_silence_duration_ms: int = 100,
|
||||
log: Optional[Callable[[str], None]] = None,
|
||||
) -> bool:
|
||||
"""Pre-STT speech gate: ``True`` only if there is at least one real speech
|
||||
region in ``audio`` (16 kHz mono float32).
|
||||
|
||||
This runs BEFORE Whisper so the model is never invoked on pure noise or a
|
||||
brief loud blip (a clap, a key clack, a mic pop) that momentarily opened the
|
||||
voice gate without anyone speaking. It uses the Silero VAD bundled with
|
||||
faster-whisper (no extra dependency). The threshold is deliberately a little
|
||||
below the faster-whisper default (0.5) so quiet but real speech is not
|
||||
dropped; precision against confident noise-hallucinations is provided by the
|
||||
downstream ``filter_speech_segments`` no_speech_prob gate.
|
||||
|
||||
Fail-open: if the VAD is unavailable or errors, return ``True`` so STT still
|
||||
runs rather than silently swallowing a real utterance.
|
||||
"""
|
||||
try:
|
||||
from faster_whisper.vad import get_speech_timestamps, VadOptions
|
||||
except Exception: # VAD not available in this build -> don't block STT
|
||||
return True
|
||||
try:
|
||||
if getattr(audio, "size", 1) == 0:
|
||||
return False
|
||||
opts = VadOptions(
|
||||
threshold=threshold,
|
||||
min_speech_duration_ms=min_speech_duration_ms,
|
||||
min_silence_duration_ms=min_silence_duration_ms,
|
||||
)
|
||||
timestamps = get_speech_timestamps(audio, opts, sampling_rate)
|
||||
return len(timestamps) > 0
|
||||
except Exception as e: # pragma: no cover - defensive
|
||||
if log:
|
||||
log(f"VAD check failed, falling back to STT: {e}")
|
||||
return True
|
||||
|
||||
|
||||
def is_non_speech(no_speech_prob: float, threshold: float) -> bool:
|
||||
"""True when Whisper flags a segment as non-speech (``>= threshold``)."""
|
||||
return no_speech_prob >= threshold
|
||||
|
||||
|
||||
def segment_confidence(seg) -> Optional[float]:
|
||||
"""Map a Whisper segment to a 0..1 confidence.
|
||||
|
||||
Prefers ``avg_logprob`` (mapped to 0..1 the same way the desktop listener
|
||||
does), falling back to ``1 - no_speech_prob`` when the log-prob is absent.
|
||||
Returns ``None`` when neither signal is available so the caller keeps the
|
||||
segment rather than dropping it on missing metadata.
|
||||
"""
|
||||
avg = getattr(seg, "avg_logprob", None)
|
||||
if avg is not None:
|
||||
return min(1.0, max(0.0, avg + 1.0))
|
||||
nsp = getattr(seg, "no_speech_prob", None)
|
||||
if nsp is not None:
|
||||
return 1.0 - nsp
|
||||
return None
|
||||
|
||||
|
||||
def filter_speech_segments(
|
||||
segments,
|
||||
*,
|
||||
no_speech_threshold: float = 0.5,
|
||||
min_confidence: float = 0.3,
|
||||
log: Optional[Callable[[str], None]] = None,
|
||||
) -> list:
|
||||
"""Keep only the segments that look like real human speech, in order.
|
||||
|
||||
``log(msg)``, if given, is called with a short reason for each dropped
|
||||
segment (used by the bridge to surface why a noisy turn produced no reply).
|
||||
"""
|
||||
kept = []
|
||||
for seg in segments:
|
||||
nsp = getattr(seg, "no_speech_prob", None)
|
||||
if nsp is not None and is_non_speech(nsp, no_speech_threshold):
|
||||
if log:
|
||||
log(f"segment dropped (no_speech_prob={nsp:.2f}): {_preview(seg)}")
|
||||
continue
|
||||
conf = segment_confidence(seg)
|
||||
if conf is not None and conf < min_confidence:
|
||||
if log:
|
||||
log(f"segment dropped (confidence={conf:.2f}): {_preview(seg)}")
|
||||
continue
|
||||
kept.append(seg)
|
||||
return kept
|
||||
|
||||
|
||||
def _preview(seg) -> str:
|
||||
return repr(getattr(seg, "text", "").strip()[:50])
|
||||
126
tests/test_bridge_stt_filter.py
Normal file
126
tests/test_bridge_stt_filter.py
Normal file
@@ -0,0 +1,126 @@
|
||||
"""Unit tests for the bridge STT speech gate.
|
||||
|
||||
The gate decides whether a Whisper segment is real human speech or just noise /
|
||||
a brief loud blip that Whisper hallucinated text from. Only speech should reach
|
||||
the reply engine, so a noisy mic that momentarily opens without anyone speaking
|
||||
produces no transcript and no reply. Thresholds are config-driven, so the tests
|
||||
pass explicit references rather than hardcoding the production defaults.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from bridge.stt_filter import (
|
||||
filter_speech_segments,
|
||||
has_speech,
|
||||
is_non_speech,
|
||||
segment_confidence,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_has_speech_rejects_silence():
|
||||
# Pure digital silence is unambiguously non-speech: VAD must skip STT.
|
||||
silence = np.zeros(16000, dtype=np.float32) # 1s @ 16kHz
|
||||
assert has_speech(silence, 16000) is False
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_has_speech_rejects_a_brief_loud_blip():
|
||||
# A short loud transient (a clap / pop): mostly silence with a 50ms spike,
|
||||
# below the min-speech duration, so no real speech region is found.
|
||||
audio = np.zeros(16000, dtype=np.float32)
|
||||
audio[8000:8800] = 0.9 # ~50ms full-scale burst
|
||||
assert has_speech(audio, 16000, min_speech_duration_ms=200) is False
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_has_speech_fails_open_on_empty_when_vad_present_returns_false():
|
||||
# Empty audio has nothing to transcribe; treat as non-speech.
|
||||
assert has_speech(np.zeros(0, dtype=np.float32), 16000) is False
|
||||
|
||||
|
||||
class Seg:
|
||||
"""Minimal stand-in for a faster-whisper segment."""
|
||||
|
||||
def __init__(self, text, no_speech_prob=0.0, avg_logprob=0.0):
|
||||
self.text = text
|
||||
self.no_speech_prob = no_speech_prob
|
||||
self.avg_logprob = avg_logprob
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_real_speech_is_kept():
|
||||
seg = Seg("오늘 일정 알려줘", no_speech_prob=0.02, avg_logprob=-0.2)
|
||||
assert filter_speech_segments(
|
||||
[seg], no_speech_threshold=0.5, min_confidence=0.3
|
||||
) == [seg]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_noise_with_high_no_speech_prob_is_dropped():
|
||||
# A mic blip Whisper hallucinated "감사합니다" from: not speech.
|
||||
seg = Seg("감사합니다", no_speech_prob=0.92, avg_logprob=0.5)
|
||||
assert filter_speech_segments(
|
||||
[seg], no_speech_threshold=0.5, min_confidence=0.3
|
||||
) == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_no_speech_cutoff_runs_before_the_confidence_check():
|
||||
# Confident hallucination: high avg_logprob but also high no_speech_prob.
|
||||
# The no-speech cutoff must catch it regardless of confidence.
|
||||
seg = Seg("MBC 뉴스", no_speech_prob=0.8, avg_logprob=0.9)
|
||||
assert filter_speech_segments(
|
||||
[seg], no_speech_threshold=0.5, min_confidence=0.3
|
||||
) == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_low_confidence_decode_is_dropped():
|
||||
# avg_logprob -0.8 -> confidence 0.2, below the 0.3 floor.
|
||||
seg = Seg("어버버", no_speech_prob=0.1, avg_logprob=-0.8)
|
||||
assert filter_speech_segments(
|
||||
[seg], no_speech_threshold=0.5, min_confidence=0.3
|
||||
) == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_order_preserved_dropping_only_non_speech():
|
||||
a = Seg("진짜 말한 문장", no_speech_prob=0.05, avg_logprob=-0.1)
|
||||
noise = Seg("음", no_speech_prob=0.95, avg_logprob=0.4)
|
||||
b = Seg("두 번째 문장", no_speech_prob=0.05, avg_logprob=-0.1)
|
||||
kept = filter_speech_segments(
|
||||
[a, noise, b], no_speech_threshold=0.5, min_confidence=0.3
|
||||
)
|
||||
assert kept == [a, b]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_segments_missing_metadata_are_kept():
|
||||
# No no_speech_prob / avg_logprob -> we can't prove it's noise, so keep it.
|
||||
class Bare:
|
||||
text = "메타데이터 없는 문장"
|
||||
|
||||
seg = Bare()
|
||||
assert filter_speech_segments(
|
||||
[seg], no_speech_threshold=0.5, min_confidence=0.3
|
||||
) == [seg]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_is_non_speech_uses_an_inclusive_threshold():
|
||||
assert is_non_speech(0.5, 0.5) is True
|
||||
assert is_non_speech(0.49, 0.5) is False
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_segment_confidence_prefers_avg_logprob():
|
||||
assert segment_confidence(Seg("x", avg_logprob=-0.2)) == pytest.approx(0.8)
|
||||
# Falls back to 1 - no_speech_prob when avg_logprob is absent.
|
||||
|
||||
class NoLogprob:
|
||||
text = "x"
|
||||
no_speech_prob = 0.25
|
||||
|
||||
assert segment_confidence(NoLogprob()) == pytest.approx(0.75)
|
||||
Reference in New Issue
Block a user