Compare commits
3 Commits
d6c029d7d5
...
6d72e10f9c
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6d72e10f9c | ||
|
|
39c7a22a12 | ||
|
|
568a1ae50b |
93
bot/src/broadcast.test.ts
Normal file
93
bot/src/broadcast.test.ts
Normal file
@@ -0,0 +1,93 @@
|
|||||||
|
import { test, expect } from "bun:test";
|
||||||
|
|
||||||
|
// broadcast.ts pulls in the runtime `config`, which requires DISCORD_GUILD_ID.
|
||||||
|
// Set it before the dynamic import so the module loads without a real .env.
|
||||||
|
process.env.DISCORD_GUILD_ID ||= "test-guild";
|
||||||
|
const { wireBroadcast, autoStartBroadcast } = await import("./broadcast.ts");
|
||||||
|
|
||||||
|
/** Minimal in-memory ScreenStreamer that records start/stop calls. */
|
||||||
|
function fakeStreamer(active = false) {
|
||||||
|
const calls: string[] = [];
|
||||||
|
let on = active;
|
||||||
|
return {
|
||||||
|
calls,
|
||||||
|
kind: "selfbot" as const,
|
||||||
|
isActive: () => on,
|
||||||
|
async start() {
|
||||||
|
calls.push("start");
|
||||||
|
on = true;
|
||||||
|
return "ok";
|
||||||
|
},
|
||||||
|
async stop() {
|
||||||
|
calls.push("stop");
|
||||||
|
on = false;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const ctx = { guildId: "g", voiceChannelId: "c" };
|
||||||
|
|
||||||
|
test("wireBroadcast reports live state to the brain", () => {
|
||||||
|
const s = fakeStreamer(false);
|
||||||
|
const session: any = {};
|
||||||
|
wireBroadcast(session, s as any, ctx);
|
||||||
|
expect(session.getBroadcasting()).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("wireBroadcast lets the brain toggle the stream by voice", async () => {
|
||||||
|
const s = fakeStreamer(false);
|
||||||
|
const session: any = {};
|
||||||
|
wireBroadcast(session, s as any, ctx);
|
||||||
|
await session.onBroadcastAction("start"); // idle -> start
|
||||||
|
expect(session.getBroadcasting()).toBe(true);
|
||||||
|
await session.onBroadcastAction("start"); // already live -> no-op
|
||||||
|
await session.onBroadcastAction("stop"); // stop
|
||||||
|
expect(session.getBroadcasting()).toBe(false);
|
||||||
|
expect(s.calls).toEqual(["start", "stop"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("autoStartBroadcast starts the broadcast when idle", async () => {
|
||||||
|
const s = fakeStreamer(false);
|
||||||
|
const ok = await autoStartBroadcast(s as any, ctx, 2, 0);
|
||||||
|
expect(ok).toBe(true);
|
||||||
|
expect(s.calls).toEqual(["start"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("autoStartBroadcast does not restart an already-live broadcast", async () => {
|
||||||
|
const s = fakeStreamer(true);
|
||||||
|
const ok = await autoStartBroadcast(s as any, ctx, 2, 0);
|
||||||
|
expect(ok).toBe(true);
|
||||||
|
expect(s.calls).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("autoStartBroadcast retries a transient failure and then succeeds", async () => {
|
||||||
|
let on = false;
|
||||||
|
const calls: string[] = [];
|
||||||
|
const s: any = {
|
||||||
|
isActive: () => on,
|
||||||
|
async start() {
|
||||||
|
calls.push("start");
|
||||||
|
if (calls.length === 1) throw new Error("transient: voice not ready");
|
||||||
|
on = true;
|
||||||
|
},
|
||||||
|
async stop() {},
|
||||||
|
};
|
||||||
|
const ok = await autoStartBroadcast(s, ctx, 2, 0);
|
||||||
|
expect(ok).toBe(true);
|
||||||
|
expect(calls).toEqual(["start", "start"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("autoStartBroadcast returns false after exhausting retries (no throw)", async () => {
|
||||||
|
let attempts = 0;
|
||||||
|
const s: any = {
|
||||||
|
isActive: () => false,
|
||||||
|
async start() {
|
||||||
|
attempts++;
|
||||||
|
throw new Error("boom");
|
||||||
|
},
|
||||||
|
async stop() {},
|
||||||
|
};
|
||||||
|
const ok = await autoStartBroadcast(s, ctx, 2, 0);
|
||||||
|
expect(ok).toBe(false);
|
||||||
|
expect(attempts).toBe(2); // retried, then gave up without throwing
|
||||||
|
});
|
||||||
121
bot/src/broadcast.ts
Normal file
121
bot/src/broadcast.ts
Normal file
@@ -0,0 +1,121 @@
|
|||||||
|
/**
|
||||||
|
* Broadcast <-> voice coupling, shared by the normal-bot and userbot entry
|
||||||
|
* points so both modes behave identically.
|
||||||
|
*
|
||||||
|
* The screen broadcast is coupled to the voice session: it auto-starts when the
|
||||||
|
* assistant joins a voice channel (screen-share mode default), reports its live
|
||||||
|
* state to the brain each turn (search routes Chrome while live / Gemini when
|
||||||
|
* off), and the brain can toggle it by voice ("방송 켜줘 / 꺼줘"). This logic
|
||||||
|
* used to live only in the normal-bot join handler, so the userbot path (the
|
||||||
|
* only mode that can actually Go Live) never started a broadcast. Keeping it
|
||||||
|
* here means a single implementation serves both.
|
||||||
|
*/
|
||||||
|
import { config } from "./config.ts";
|
||||||
|
import { createStreamer, type ScreenStreamer, type StreamContext } from "./stream/index.ts";
|
||||||
|
import type { VoiceSession } from "./voice.ts";
|
||||||
|
|
||||||
|
/** One streamer per guild, shared across both entry points. */
|
||||||
|
const streamers = new Map<string, ScreenStreamer>();
|
||||||
|
|
||||||
|
export async function getStreamer(guildId: string): Promise<ScreenStreamer> {
|
||||||
|
let s = streamers.get(guildId);
|
||||||
|
if (!s) {
|
||||||
|
s = await createStreamer(config);
|
||||||
|
streamers.set(guildId, s);
|
||||||
|
}
|
||||||
|
return s;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The existing streamer for a guild, if one has been created. */
|
||||||
|
export function peekStreamer(guildId: string): ScreenStreamer | undefined {
|
||||||
|
return streamers.get(guildId);
|
||||||
|
}
|
||||||
|
|
||||||
|
type BroadcastSession = Pick<VoiceSession, "getBroadcasting" | "onBroadcastAction">;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wire a voice session to a streamer (no auto-start). Pure (no config / registry
|
||||||
|
* access) so it can be unit-tested with a fake streamer:
|
||||||
|
* - report live state to the brain each turn,
|
||||||
|
* - let the brain toggle the stream by voice.
|
||||||
|
*/
|
||||||
|
export function wireBroadcast(
|
||||||
|
session: BroadcastSession,
|
||||||
|
streamer: ScreenStreamer,
|
||||||
|
ctx: StreamContext,
|
||||||
|
): void {
|
||||||
|
session.getBroadcasting = () => streamer.isActive();
|
||||||
|
session.onBroadcastAction = async (action) => {
|
||||||
|
if (action === "start") {
|
||||||
|
if (!streamer.isActive()) await streamer.start(ctx);
|
||||||
|
} else if (streamer.isActive()) {
|
||||||
|
await streamer.stop();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Auto-start the broadcast, retrying a couple of times before giving up. The
|
||||||
|
* selfbot Go-Live path takes ~10-15s of humanised delays and can fail on a
|
||||||
|
* transient (login race, voice not ready yet), so a bounded retry recovers the
|
||||||
|
* common case. On final failure it logs loudly rather than silently leaving the
|
||||||
|
* user in voice with no broadcast. Never throws: a broadcast problem must not
|
||||||
|
* tear down the voice conversation. Returns whether the stream ended up live.
|
||||||
|
*/
|
||||||
|
export async function autoStartBroadcast(
|
||||||
|
streamer: ScreenStreamer,
|
||||||
|
ctx: StreamContext,
|
||||||
|
attempts = 2,
|
||||||
|
retryDelayMs = 1500,
|
||||||
|
): Promise<boolean> {
|
||||||
|
for (let attempt = 1; attempt <= attempts; attempt++) {
|
||||||
|
try {
|
||||||
|
if (streamer.isActive()) return true;
|
||||||
|
await streamer.start(ctx);
|
||||||
|
if (streamer.isActive()) return true;
|
||||||
|
console.error(
|
||||||
|
`[broadcast] auto-start attempt ${attempt}/${attempts} did not go live`,
|
||||||
|
);
|
||||||
|
} catch (e) {
|
||||||
|
console.error(`[broadcast] auto-start attempt ${attempt}/${attempts} failed:`, e);
|
||||||
|
}
|
||||||
|
if (attempt < attempts && retryDelayMs > 0) {
|
||||||
|
await new Promise((r) => setTimeout(r, retryDelayMs));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.error(
|
||||||
|
"[broadcast] ⚠️ auto-start FAILED after retries — voice is up but NOT broadcasting " +
|
||||||
|
"(STREAM_BROWSER=true). Trigger it by voice ('방송 켜줘') or check the selfbot stream deps.",
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Couple the broadcast to a freshly joined voice session when screen-share mode
|
||||||
|
* is on (STREAM_BROWSER=true). No-op in voice-only mode (STREAM_BROWSER=false).
|
||||||
|
*
|
||||||
|
* The toggle hooks are wired synchronously; the auto-start runs in the
|
||||||
|
* BACKGROUND so the ~10-15s Go-Live handshake never blocks the voice session
|
||||||
|
* from coming up. The bot can already hear and reply while the stream warms.
|
||||||
|
*/
|
||||||
|
export async function maybeCoupleBroadcast(
|
||||||
|
session: BroadcastSession,
|
||||||
|
ctx: StreamContext,
|
||||||
|
): Promise<void> {
|
||||||
|
if (!config.screenBrowser) return;
|
||||||
|
const streamer = await getStreamer(ctx.guildId);
|
||||||
|
wireBroadcast(session, streamer, ctx);
|
||||||
|
void autoStartBroadcast(streamer, ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Stop the broadcast for a guild if one is live (e.g. on leave). */
|
||||||
|
export async function stopBroadcast(guildId: string): Promise<void> {
|
||||||
|
const s = streamers.get(guildId);
|
||||||
|
if (s?.isActive()) {
|
||||||
|
try {
|
||||||
|
await s.stop();
|
||||||
|
} catch (e) {
|
||||||
|
console.error("[broadcast] stop failed:", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -19,23 +19,13 @@ import { AttachmentBuilder } from "discord.js";
|
|||||||
import { config } from "./config.ts";
|
import { config } from "./config.ts";
|
||||||
import { ask, health } from "./bridge.ts";
|
import { ask, health } from "./bridge.ts";
|
||||||
import { joinChannel, leaveGuild, getSession } from "./voice.ts";
|
import { joinChannel, leaveGuild, getSession } from "./voice.ts";
|
||||||
import { createStreamer, type ScreenStreamer, type StreamContext } from "./stream/index.ts";
|
import { type StreamContext } from "./stream/index.ts";
|
||||||
|
import { getStreamer, peekStreamer, maybeCoupleBroadcast, stopBroadcast } from "./broadcast.ts";
|
||||||
|
|
||||||
const client = new Client({
|
const client = new Client({
|
||||||
intents: [GatewayIntentBits.Guilds, GatewayIntentBits.GuildVoiceStates],
|
intents: [GatewayIntentBits.Guilds, GatewayIntentBits.GuildVoiceStates],
|
||||||
});
|
});
|
||||||
|
|
||||||
const streamers = new Map<string, ScreenStreamer>();
|
|
||||||
|
|
||||||
async function getStreamer(guildId: string): Promise<ScreenStreamer> {
|
|
||||||
let s = streamers.get(guildId);
|
|
||||||
if (!s) {
|
|
||||||
s = await createStreamer(config);
|
|
||||||
streamers.set(guildId, s);
|
|
||||||
}
|
|
||||||
return s;
|
|
||||||
}
|
|
||||||
|
|
||||||
const eph = { flags: MessageFlags.Ephemeral } as const;
|
const eph = { flags: MessageFlags.Ephemeral } as const;
|
||||||
|
|
||||||
client.once("clientReady", () => {
|
client.once("clientReady", () => {
|
||||||
@@ -87,23 +77,7 @@ async function handleJoin(i: ChatInputCommandInteraction) {
|
|||||||
// each turn (search routes Chrome while live / Gemini when off), and let the
|
// each turn (search routes Chrome while live / Gemini when off), and let the
|
||||||
// brain toggle it via voice ("방송 켜줘 / 꺼줘"). In voice-only mode
|
// brain toggle it via voice ("방송 켜줘 / 꺼줘"). In voice-only mode
|
||||||
// (STREAM_BROWSER=false) none of this runs and the broadcast stays off.
|
// (STREAM_BROWSER=false) none of this runs and the broadcast stays off.
|
||||||
if (config.screenBrowser) {
|
await maybeCoupleBroadcast(session, { guildId: i.guildId!, voiceChannelId: channel.id });
|
||||||
const streamer = await getStreamer(i.guildId!);
|
|
||||||
const ctx: StreamContext = { guildId: i.guildId!, voiceChannelId: channel.id };
|
|
||||||
session.getBroadcasting = () => streamer.isActive();
|
|
||||||
session.onBroadcastAction = async (action) => {
|
|
||||||
if (action === "start") {
|
|
||||||
if (!streamer.isActive()) await streamer.start(ctx);
|
|
||||||
} else if (streamer.isActive()) {
|
|
||||||
await streamer.stop();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
try {
|
|
||||||
if (!streamer.isActive()) await streamer.start(ctx);
|
|
||||||
} catch (e) {
|
|
||||||
console.error("[join] auto-broadcast failed:", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
await i.editReply(`🎙️ '${channel.name}' 채널에 접속했습니다. 말씀하세요.`);
|
await i.editReply(`🎙️ '${channel.name}' 채널에 접속했습니다. 말씀하세요.`);
|
||||||
}
|
}
|
||||||
@@ -112,14 +86,7 @@ async function handleLeave(i: ChatInputCommandInteraction) {
|
|||||||
const left = leaveGuild(i.guildId!);
|
const left = leaveGuild(i.guildId!);
|
||||||
// Leaving voice also ends the broadcast — don't leave a stream live with no
|
// Leaving voice also ends the broadcast — don't leave a stream live with no
|
||||||
// session driving it.
|
// session driving it.
|
||||||
const streamer = streamers.get(i.guildId!);
|
await stopBroadcast(i.guildId!);
|
||||||
if (streamer?.isActive()) {
|
|
||||||
try {
|
|
||||||
await streamer.stop();
|
|
||||||
} catch (e) {
|
|
||||||
console.error("[leave] stopping broadcast failed:", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await i.reply({ content: left ? "음성 채널에서 나갔습니다." : "접속 중인 세션이 없습니다.", ...eph });
|
await i.reply({ content: left ? "음성 채널에서 나갔습니다." : "접속 중인 세션이 없습니다.", ...eph });
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -158,7 +125,7 @@ async function handleStream(i: ChatInputCommandInteraction) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function handleStop(i: ChatInputCommandInteraction) {
|
async function handleStop(i: ChatInputCommandInteraction) {
|
||||||
const streamer = streamers.get(i.guildId!);
|
const streamer = peekStreamer(i.guildId!);
|
||||||
if (!streamer) return i.reply({ content: "송출 중이 아닙니다.", ...eph });
|
if (!streamer) return i.reply({ content: "송출 중이 아닙니다.", ...eph });
|
||||||
await streamer.stop();
|
await streamer.stop();
|
||||||
await i.reply({ content: "송출을 중단했습니다.", ...eph });
|
await i.reply({ content: "송출을 중단했습니다.", ...eph });
|
||||||
@@ -174,7 +141,7 @@ async function handleStatus(i: ChatInputCommandInteraction) {
|
|||||||
/* keep unreachable */
|
/* keep unreachable */
|
||||||
}
|
}
|
||||||
const session = getSession(i.guildId!);
|
const session = getSession(i.guildId!);
|
||||||
const streamer = streamers.get(i.guildId!);
|
const streamer = peekStreamer(i.guildId!);
|
||||||
await i.editReply(
|
await i.editReply(
|
||||||
[
|
[
|
||||||
`브릿지 두뇌: ${brain}`,
|
`브릿지 두뇌: ${brain}`,
|
||||||
|
|||||||
@@ -19,6 +19,7 @@
|
|||||||
import type { VoiceBasedChannel } from "discord.js";
|
import type { VoiceBasedChannel } from "discord.js";
|
||||||
import { config } from "./config.ts";
|
import { config } from "./config.ts";
|
||||||
import { joinChannel, leaveGuild } from "./voice.ts";
|
import { joinChannel, leaveGuild } from "./voice.ts";
|
||||||
|
import { maybeCoupleBroadcast, stopBroadcast } from "./broadcast.ts";
|
||||||
|
|
||||||
type AnyClient = any;
|
type AnyClient = any;
|
||||||
|
|
||||||
@@ -43,6 +44,11 @@ async function joinAndListen(client: AnyClient, channelId: string): Promise<void
|
|||||||
// joinVoiceChannel (it exposes id, guild.id and guild.voiceAdapterCreator).
|
// joinVoiceChannel (it exposes id, guild.id and guild.voiceAdapterCreator).
|
||||||
const session = await joinChannel(channel as unknown as VoiceBasedChannel);
|
const session = await joinChannel(channel as unknown as VoiceBasedChannel);
|
||||||
session.onTurn = ({ transcript, reply }) => console.log(`🗣️ ${transcript}\n🤖 ${reply}`);
|
session.onTurn = ({ transcript, reply }) => console.log(`🗣️ ${transcript}\n🤖 ${reply}`);
|
||||||
|
// Screen-share mode (STREAM_BROWSER=true): auto-start the broadcast on join,
|
||||||
|
// report its live state to the brain each turn, and let the brain toggle it by
|
||||||
|
// voice. Userbot is the only mode that can actually Go Live, so without this
|
||||||
|
// wiring the broadcast never starts. No-op when STREAM_BROWSER=false.
|
||||||
|
await maybeCoupleBroadcast(session, { guildId: channel.guild.id, voiceChannelId: channel.id });
|
||||||
console.log(`🎙️ 유저봇이 '${channel.name}' 음성채널에 참여했습니다.`);
|
console.log(`🎙️ 유저봇이 '${channel.name}' 음성채널에 참여했습니다.`);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -72,6 +78,9 @@ export async function runUserbot(): Promise<void> {
|
|||||||
const ch = msg.member?.voice?.channel || msg.author?.voice?.channel;
|
const ch = msg.member?.voice?.channel || msg.author?.voice?.channel;
|
||||||
if (ch) await joinAndListen(client, ch.id).catch((e) => console.error("[userbot] join cmd:", e));
|
if (ch) await joinAndListen(client, ch.id).catch((e) => console.error("[userbot] join cmd:", e));
|
||||||
} else if (content === "!자비스 leave" || content === "!jarvis leave") {
|
} else if (content === "!자비스 leave" || content === "!jarvis leave") {
|
||||||
|
// Leaving voice also ends the broadcast — don't leave a stream live with
|
||||||
|
// no session driving it.
|
||||||
|
await stopBroadcast(config.guildId);
|
||||||
leaveGuild(config.guildId);
|
leaveGuild(config.guildId);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -51,8 +51,10 @@ from flask import Flask, request, jsonify, Response, stream_with_context
|
|||||||
|
|
||||||
try: # package-relative when imported as ``bridge.server``
|
try: # package-relative when imported as ``bridge.server``
|
||||||
from bridge.text_utils import split_sentences
|
from bridge.text_utils import split_sentences
|
||||||
|
from bridge.stt_filter import filter_speech_segments, has_speech
|
||||||
except ImportError: # script-relative when run as ``bridge/server.py``
|
except ImportError: # script-relative when run as ``bridge/server.py``
|
||||||
from text_utils import split_sentences
|
from text_utils import split_sentences
|
||||||
|
from stt_filter import filter_speech_segments, has_speech
|
||||||
|
|
||||||
app = Flask(__name__)
|
app = Flask(__name__)
|
||||||
|
|
||||||
@@ -64,6 +66,15 @@ BRIDGE_PORT = int(os.environ.get("BRIDGE_PORT", "8765"))
|
|||||||
BRAIN_ENABLED = os.environ.get("JARVIS_BRAIN_ENABLED", "1") not in ("0", "false", "False")
|
BRAIN_ENABLED = os.environ.get("JARVIS_BRAIN_ENABLED", "1") not in ("0", "false", "False")
|
||||||
TTS_ENABLED = os.environ.get("JARVIS_TTS_ENABLED", "1") not in ("0", "false", "False")
|
TTS_ENABLED = os.environ.get("JARVIS_TTS_ENABLED", "1") not in ("0", "false", "False")
|
||||||
|
|
||||||
|
# Pre-STT speech gate (Silero VAD). Tunable for the Discord mic without a code
|
||||||
|
# change: raise VAD_THRESHOLD to reject more noise, lower it to catch quieter
|
||||||
|
# speech. VAD_MIN_SPEECH_MS is the shortest run of speech that counts (a brief
|
||||||
|
# loud blip shorter than this never reaches Whisper). Set VAD_ENABLED=0 to fall
|
||||||
|
# back to the old behaviour (always transcribe, rely on the post-filter only).
|
||||||
|
VAD_ENABLED = os.environ.get("VAD_ENABLED", "1") not in ("0", "false", "False")
|
||||||
|
VAD_THRESHOLD = float(os.environ.get("VAD_THRESHOLD", "0.4"))
|
||||||
|
VAD_MIN_SPEECH_MS = int(os.environ.get("VAD_MIN_SPEECH_MS", "200"))
|
||||||
|
|
||||||
# TTS engine: "melo" (MeloTTS Korean speaker, the warm worker) is the primary
|
# TTS engine: "melo" (MeloTTS Korean speaker, the warm worker) is the primary
|
||||||
# voice; Piper is kept as a fallback if the worker is unreachable. Set
|
# voice; Piper is kept as a fallback if the worker is unreachable. Set
|
||||||
# TTS_ENGINE=piper to disable MeloTTS entirely.
|
# TTS_ENGINE=piper to disable MeloTTS entirely.
|
||||||
@@ -182,8 +193,35 @@ def transcribe(wav_bytes: bytes) -> dict:
|
|||||||
x_old = np.linspace(0.0, 1.0, num=audio.size, endpoint=False)
|
x_old = np.linspace(0.0, 1.0, num=audio.size, endpoint=False)
|
||||||
x_new = np.linspace(0.0, 1.0, num=n_out, endpoint=False)
|
x_new = np.linspace(0.0, 1.0, num=n_out, endpoint=False)
|
||||||
audio = np.interp(x_new, x_old, audio).astype(np.float32)
|
audio = np.interp(x_new, x_old, audio).astype(np.float32)
|
||||||
|
# Pre-STT speech gate: don't even invoke Whisper unless there is real speech
|
||||||
|
# in the clip. Noise or a brief loud blip (no actual speech) is dropped here,
|
||||||
|
# before transcription, so the model never gets a chance to hallucinate a
|
||||||
|
# phrase from it. Fail-open inside has_speech() keeps a real utterance from
|
||||||
|
# being swallowed if the VAD is unavailable.
|
||||||
|
if VAD_ENABLED and not has_speech(
|
||||||
|
audio,
|
||||||
|
16000,
|
||||||
|
threshold=VAD_THRESHOLD,
|
||||||
|
min_speech_duration_ms=VAD_MIN_SPEECH_MS,
|
||||||
|
log=lambda m: print(f"[bridge] {m}", flush=True),
|
||||||
|
):
|
||||||
|
print("[bridge] no speech detected (VAD) — skipping STT", flush=True)
|
||||||
|
return {"text": "", "language": None}
|
||||||
|
|
||||||
segments, info = _whisper.transcribe(audio, beam_size=1)
|
segments, info = _whisper.transcribe(audio, beam_size=1)
|
||||||
text = "".join(seg.text for seg in segments).strip()
|
# Second line of defence: even speech-like audio (e.g. a clap the VAD let
|
||||||
|
# through) can make Whisper hallucinate, so drop non-speech / low-confidence
|
||||||
|
# segments by Whisper's own no_speech_prob + avg_logprob. Mirrors the desktop
|
||||||
|
# listener's policy, driven by the same config thresholds.
|
||||||
|
no_speech_threshold = getattr(_cfg, "whisper_no_speech_threshold", 0.5)
|
||||||
|
min_confidence = getattr(_cfg, "whisper_min_confidence", 0.3)
|
||||||
|
kept = filter_speech_segments(
|
||||||
|
segments,
|
||||||
|
no_speech_threshold=no_speech_threshold,
|
||||||
|
min_confidence=min_confidence,
|
||||||
|
log=lambda m: print(f"[bridge] {m}", flush=True),
|
||||||
|
)
|
||||||
|
text = "".join(seg.text for seg in kept).strip()
|
||||||
return {"text": text, "language": getattr(info, "language", None)}
|
return {"text": text, "language": getattr(info, "language", None)}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
122
bridge/stt_filter.py
Normal file
122
bridge/stt_filter.py
Normal file
@@ -0,0 +1,122 @@
|
|||||||
|
"""Speech gate for the Discord STT path.
|
||||||
|
|
||||||
|
Whisper will transcribe, and frequently *hallucinate*, on non-speech audio:
|
||||||
|
silence, background noise, or a brief loud blip (a cough, a key clack, a mic
|
||||||
|
pop) that momentarily opens the voice gate without anyone actually speaking.
|
||||||
|
Left unfiltered those produce phantom transcripts ("MBC 뉴스", "감사합니다", ...)
|
||||||
|
and the assistant ends up replying to noise.
|
||||||
|
|
||||||
|
This mirrors the desktop listener's ``_filter_noisy_segments`` policy
|
||||||
|
(``src/jarvis/listening/listener.py``) so both entry points apply identical
|
||||||
|
rules, both driven by the same config thresholds:
|
||||||
|
|
||||||
|
1. Hard ``no_speech_prob`` cutoff (``whisper_no_speech_threshold``): Whisper's
|
||||||
|
own "this segment is not speech" probability. Checked first and
|
||||||
|
independently of confidence, because Whisper can be *confident* about a
|
||||||
|
hallucinated phrase on pure noise.
|
||||||
|
2. ``avg_logprob`` confidence floor (``whisper_min_confidence``): drops
|
||||||
|
low-quality decodes that survive the no-speech check.
|
||||||
|
|
||||||
|
A segment must pass both to count as real human speech.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Callable, Optional
|
||||||
|
|
||||||
|
|
||||||
|
def has_speech(
|
||||||
|
audio,
|
||||||
|
sampling_rate: int = 16000,
|
||||||
|
*,
|
||||||
|
threshold: float = 0.4,
|
||||||
|
min_speech_duration_ms: int = 200,
|
||||||
|
min_silence_duration_ms: int = 100,
|
||||||
|
log: Optional[Callable[[str], None]] = None,
|
||||||
|
) -> bool:
|
||||||
|
"""Pre-STT speech gate: ``True`` only if there is at least one real speech
|
||||||
|
region in ``audio`` (16 kHz mono float32).
|
||||||
|
|
||||||
|
This runs BEFORE Whisper so the model is never invoked on pure noise or a
|
||||||
|
brief loud blip (a clap, a key clack, a mic pop) that momentarily opened the
|
||||||
|
voice gate without anyone speaking. It uses the Silero VAD bundled with
|
||||||
|
faster-whisper (no extra dependency). The threshold is deliberately a little
|
||||||
|
below the faster-whisper default (0.5) so quiet but real speech is not
|
||||||
|
dropped; precision against confident noise-hallucinations is provided by the
|
||||||
|
downstream ``filter_speech_segments`` no_speech_prob gate.
|
||||||
|
|
||||||
|
Fail-open: if the VAD is unavailable or errors, return ``True`` so STT still
|
||||||
|
runs rather than silently swallowing a real utterance.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
from faster_whisper.vad import get_speech_timestamps, VadOptions
|
||||||
|
except Exception: # VAD not available in this build -> don't block STT
|
||||||
|
return True
|
||||||
|
try:
|
||||||
|
if getattr(audio, "size", 1) == 0:
|
||||||
|
return False
|
||||||
|
opts = VadOptions(
|
||||||
|
threshold=threshold,
|
||||||
|
min_speech_duration_ms=min_speech_duration_ms,
|
||||||
|
min_silence_duration_ms=min_silence_duration_ms,
|
||||||
|
)
|
||||||
|
timestamps = get_speech_timestamps(audio, opts, sampling_rate)
|
||||||
|
return len(timestamps) > 0
|
||||||
|
except Exception as e: # pragma: no cover - defensive
|
||||||
|
if log:
|
||||||
|
log(f"VAD check failed, falling back to STT: {e}")
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def is_non_speech(no_speech_prob: float, threshold: float) -> bool:
|
||||||
|
"""True when Whisper flags a segment as non-speech (``>= threshold``)."""
|
||||||
|
return no_speech_prob >= threshold
|
||||||
|
|
||||||
|
|
||||||
|
def segment_confidence(seg) -> Optional[float]:
|
||||||
|
"""Map a Whisper segment to a 0..1 confidence.
|
||||||
|
|
||||||
|
Prefers ``avg_logprob`` (mapped to 0..1 the same way the desktop listener
|
||||||
|
does), falling back to ``1 - no_speech_prob`` when the log-prob is absent.
|
||||||
|
Returns ``None`` when neither signal is available so the caller keeps the
|
||||||
|
segment rather than dropping it on missing metadata.
|
||||||
|
"""
|
||||||
|
avg = getattr(seg, "avg_logprob", None)
|
||||||
|
if avg is not None:
|
||||||
|
return min(1.0, max(0.0, avg + 1.0))
|
||||||
|
nsp = getattr(seg, "no_speech_prob", None)
|
||||||
|
if nsp is not None:
|
||||||
|
return 1.0 - nsp
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def filter_speech_segments(
|
||||||
|
segments,
|
||||||
|
*,
|
||||||
|
no_speech_threshold: float = 0.5,
|
||||||
|
min_confidence: float = 0.3,
|
||||||
|
log: Optional[Callable[[str], None]] = None,
|
||||||
|
) -> list:
|
||||||
|
"""Keep only the segments that look like real human speech, in order.
|
||||||
|
|
||||||
|
``log(msg)``, if given, is called with a short reason for each dropped
|
||||||
|
segment (used by the bridge to surface why a noisy turn produced no reply).
|
||||||
|
"""
|
||||||
|
kept = []
|
||||||
|
for seg in segments:
|
||||||
|
nsp = getattr(seg, "no_speech_prob", None)
|
||||||
|
if nsp is not None and is_non_speech(nsp, no_speech_threshold):
|
||||||
|
if log:
|
||||||
|
log(f"segment dropped (no_speech_prob={nsp:.2f}): {_preview(seg)}")
|
||||||
|
continue
|
||||||
|
conf = segment_confidence(seg)
|
||||||
|
if conf is not None and conf < min_confidence:
|
||||||
|
if log:
|
||||||
|
log(f"segment dropped (confidence={conf:.2f}): {_preview(seg)}")
|
||||||
|
continue
|
||||||
|
kept.append(seg)
|
||||||
|
return kept
|
||||||
|
|
||||||
|
|
||||||
|
def _preview(seg) -> str:
|
||||||
|
return repr(getattr(seg, "text", "").strip()[:50])
|
||||||
126
tests/test_bridge_stt_filter.py
Normal file
126
tests/test_bridge_stt_filter.py
Normal file
@@ -0,0 +1,126 @@
|
|||||||
|
"""Unit tests for the bridge STT speech gate.
|
||||||
|
|
||||||
|
The gate decides whether a Whisper segment is real human speech or just noise /
|
||||||
|
a brief loud blip that Whisper hallucinated text from. Only speech should reach
|
||||||
|
the reply engine, so a noisy mic that momentarily opens without anyone speaking
|
||||||
|
produces no transcript and no reply. Thresholds are config-driven, so the tests
|
||||||
|
pass explicit references rather than hardcoding the production defaults.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from bridge.stt_filter import (
|
||||||
|
filter_speech_segments,
|
||||||
|
has_speech,
|
||||||
|
is_non_speech,
|
||||||
|
segment_confidence,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_has_speech_rejects_silence():
|
||||||
|
# Pure digital silence is unambiguously non-speech: VAD must skip STT.
|
||||||
|
silence = np.zeros(16000, dtype=np.float32) # 1s @ 16kHz
|
||||||
|
assert has_speech(silence, 16000) is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_has_speech_rejects_a_brief_loud_blip():
|
||||||
|
# A short loud transient (a clap / pop): mostly silence with a 50ms spike,
|
||||||
|
# below the min-speech duration, so no real speech region is found.
|
||||||
|
audio = np.zeros(16000, dtype=np.float32)
|
||||||
|
audio[8000:8800] = 0.9 # ~50ms full-scale burst
|
||||||
|
assert has_speech(audio, 16000, min_speech_duration_ms=200) is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_has_speech_fails_open_on_empty_when_vad_present_returns_false():
|
||||||
|
# Empty audio has nothing to transcribe; treat as non-speech.
|
||||||
|
assert has_speech(np.zeros(0, dtype=np.float32), 16000) is False
|
||||||
|
|
||||||
|
|
||||||
|
class Seg:
|
||||||
|
"""Minimal stand-in for a faster-whisper segment."""
|
||||||
|
|
||||||
|
def __init__(self, text, no_speech_prob=0.0, avg_logprob=0.0):
|
||||||
|
self.text = text
|
||||||
|
self.no_speech_prob = no_speech_prob
|
||||||
|
self.avg_logprob = avg_logprob
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_real_speech_is_kept():
|
||||||
|
seg = Seg("오늘 일정 알려줘", no_speech_prob=0.02, avg_logprob=-0.2)
|
||||||
|
assert filter_speech_segments(
|
||||||
|
[seg], no_speech_threshold=0.5, min_confidence=0.3
|
||||||
|
) == [seg]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_noise_with_high_no_speech_prob_is_dropped():
|
||||||
|
# A mic blip Whisper hallucinated "감사합니다" from: not speech.
|
||||||
|
seg = Seg("감사합니다", no_speech_prob=0.92, avg_logprob=0.5)
|
||||||
|
assert filter_speech_segments(
|
||||||
|
[seg], no_speech_threshold=0.5, min_confidence=0.3
|
||||||
|
) == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_no_speech_cutoff_runs_before_the_confidence_check():
|
||||||
|
# Confident hallucination: high avg_logprob but also high no_speech_prob.
|
||||||
|
# The no-speech cutoff must catch it regardless of confidence.
|
||||||
|
seg = Seg("MBC 뉴스", no_speech_prob=0.8, avg_logprob=0.9)
|
||||||
|
assert filter_speech_segments(
|
||||||
|
[seg], no_speech_threshold=0.5, min_confidence=0.3
|
||||||
|
) == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_low_confidence_decode_is_dropped():
|
||||||
|
# avg_logprob -0.8 -> confidence 0.2, below the 0.3 floor.
|
||||||
|
seg = Seg("어버버", no_speech_prob=0.1, avg_logprob=-0.8)
|
||||||
|
assert filter_speech_segments(
|
||||||
|
[seg], no_speech_threshold=0.5, min_confidence=0.3
|
||||||
|
) == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_order_preserved_dropping_only_non_speech():
|
||||||
|
a = Seg("진짜 말한 문장", no_speech_prob=0.05, avg_logprob=-0.1)
|
||||||
|
noise = Seg("음", no_speech_prob=0.95, avg_logprob=0.4)
|
||||||
|
b = Seg("두 번째 문장", no_speech_prob=0.05, avg_logprob=-0.1)
|
||||||
|
kept = filter_speech_segments(
|
||||||
|
[a, noise, b], no_speech_threshold=0.5, min_confidence=0.3
|
||||||
|
)
|
||||||
|
assert kept == [a, b]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_segments_missing_metadata_are_kept():
|
||||||
|
# No no_speech_prob / avg_logprob -> we can't prove it's noise, so keep it.
|
||||||
|
class Bare:
|
||||||
|
text = "메타데이터 없는 문장"
|
||||||
|
|
||||||
|
seg = Bare()
|
||||||
|
assert filter_speech_segments(
|
||||||
|
[seg], no_speech_threshold=0.5, min_confidence=0.3
|
||||||
|
) == [seg]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_is_non_speech_uses_an_inclusive_threshold():
|
||||||
|
assert is_non_speech(0.5, 0.5) is True
|
||||||
|
assert is_non_speech(0.49, 0.5) is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_segment_confidence_prefers_avg_logprob():
|
||||||
|
assert segment_confidence(Seg("x", avg_logprob=-0.2)) == pytest.approx(0.8)
|
||||||
|
# Falls back to 1 - no_speech_prob when avg_logprob is absent.
|
||||||
|
|
||||||
|
class NoLogprob:
|
||||||
|
text = "x"
|
||||||
|
no_speech_prob = 0.25
|
||||||
|
|
||||||
|
assert segment_confidence(NoLogprob()) == pytest.approx(0.75)
|
||||||
Reference in New Issue
Block a user