feat(stt-log): log the WHOLE turn pipeline to the transcript channel
The transcript channel only showed successful transcripts, so dropped utterances (the 47/50 misses) were invisible. Now every captured utterance is mirrored with its outcome and per-stage timing: - too-short blip (<300ms), VAD "음성 아님(VAD 차단)", "인식 실패", "답변 없음", or "ok" - transcript + reply (or "(무응답)") - ⏱️ stt/llm seconds The bridge meta now carries note + stt_sec + think_sec; voice.ts fires onTurn for every turn (not only non-empty transcripts) and for the too-short drop; userbot formats the diagnostic line. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -45,6 +45,11 @@ export interface ConverseMeta {
|
||||
reply: string;
|
||||
error?: string | null;
|
||||
broadcast_action?: BroadcastAction | null;
|
||||
/** Why this turn produced (or didn't produce) a transcript/reply. */
|
||||
note?: string;
|
||||
/** Per-stage timing (seconds) for diagnosing latency. */
|
||||
stt_sec?: number;
|
||||
think_sec?: number;
|
||||
}
|
||||
|
||||
export interface ConverseStreamHandlers {
|
||||
|
||||
@@ -34,16 +34,33 @@ async function loadSelfbot(): Promise<any> {
|
||||
}
|
||||
}
|
||||
|
||||
/** Mirror a heard voice turn (transcript + reply) into a text channel. */
|
||||
async function postTranscript(client: AnyClient, transcript: string, reply: string): Promise<void> {
|
||||
interface TurnInfo {
|
||||
transcript: string;
|
||||
reply: string;
|
||||
note?: string;
|
||||
sttSec?: number;
|
||||
thinkSec?: number;
|
||||
}
|
||||
|
||||
/** Mirror EVERY heard utterance (and why it did/didn't answer) to a text
|
||||
* channel, so misses and latency are diagnosable without hearing the bot. */
|
||||
async function postTranscript(client: AnyClient, info: TurnInfo): Promise<void> {
|
||||
const chId = config.transcriptChannelId;
|
||||
if (!chId) return;
|
||||
const timing =
|
||||
info.sttSec != null || info.thinkSec != null
|
||||
? ` ⏱️ stt ${info.sttSec ?? "?"}s · llm ${info.thinkSec ?? "?"}s`
|
||||
: "";
|
||||
let msg: string;
|
||||
if (info.transcript) {
|
||||
const r = (info.reply || "").trim();
|
||||
msg = `🎤 들음 → 🗣️ "${info.transcript}"\n🤖 답변: ${r || "(무응답)"}${timing}`;
|
||||
} else {
|
||||
msg = `🎤 들음 → ❌ ${info.note || "무시됨"}${timing}`;
|
||||
}
|
||||
try {
|
||||
const ch: any = await client.channels.fetch(chId).catch(() => null);
|
||||
if (ch?.send) {
|
||||
const r = (reply || "").trim();
|
||||
await ch.send(`🗣️ 들은 말: ${transcript}\n🤖 답변: ${r || "(무응답)"}`);
|
||||
}
|
||||
if (ch?.send) await ch.send(msg);
|
||||
} catch (e) {
|
||||
console.error("[userbot] transcript post failed:", e);
|
||||
}
|
||||
@@ -58,11 +75,11 @@ async function joinAndListen(client: AnyClient, channelId: string): Promise<void
|
||||
// The selfbot VoiceChannel is runtime-compatible with @discordjs/voice's
|
||||
// joinVoiceChannel (it exposes id, guild.id and guild.voiceAdapterCreator).
|
||||
const session = await joinChannel(channel as unknown as VoiceBasedChannel);
|
||||
session.onTurn = ({ transcript, reply }) => {
|
||||
console.log(`🗣️ ${transcript}\n🤖 ${reply}`);
|
||||
// Mirror every heard utterance (and the reply, if any) to a text channel so
|
||||
// you can see what the bot understood even when it doesn't answer aloud.
|
||||
void postTranscript(client, transcript, reply);
|
||||
session.onTurn = (info) => {
|
||||
console.log(`🗣️ ${info.transcript || "(" + (info.note || "empty") + ")"}\n🤖 ${info.reply}`);
|
||||
// Mirror every heard utterance (and the reply / drop reason) to a text
|
||||
// channel so you can see what the bot understood even when it doesn't answer.
|
||||
void postTranscript(client, info);
|
||||
};
|
||||
// Screen-share mode (STREAM_BROWSER=true): auto-start the broadcast on join,
|
||||
// report its live state to the brain each turn, and let the brain toggle it by
|
||||
|
||||
@@ -69,8 +69,17 @@ export class VoiceSession {
|
||||
/** Pending reply clips. Played one at a time so concurrent speakers don't
|
||||
* cut each other's replies off. */
|
||||
private playQueue: Buffer[] = [];
|
||||
/** Optional callback to surface transcripts/replies to a text channel. */
|
||||
onTurn?: (info: { user: string; transcript: string; reply: string }) => void;
|
||||
/** Optional callback to surface EVERY heard utterance (and its outcome) to a
|
||||
* text channel — including ones dropped before/at STT — so misses are
|
||||
* diagnosable. `note` says why (e.g. "음성 아님(VAD 차단)", "너무 짧음", "ok"). */
|
||||
onTurn?: (info: {
|
||||
user: string;
|
||||
transcript: string;
|
||||
reply: string;
|
||||
note?: string;
|
||||
sttSec?: number;
|
||||
thinkSec?: number;
|
||||
}) => void;
|
||||
/** Live screen-share state, sent with each turn so the brain routes search
|
||||
* (Chrome while broadcasting, Gemini when off). */
|
||||
getBroadcasting?: () => boolean;
|
||||
@@ -123,8 +132,12 @@ export class VoiceSession {
|
||||
|
||||
if (!chunks.length) return;
|
||||
const mono = stereoToMono(Buffer.concat(chunks));
|
||||
// Ignore blips shorter than ~300ms (likely noise / key clicks).
|
||||
if (mono.length < DISCORD_RATE * 0.3 * 2) return;
|
||||
// Ignore blips shorter than ~300ms (likely noise / key clicks) — but still
|
||||
// report them so the transcript channel shows every captured utterance.
|
||||
if (mono.length < DISCORD_RATE * 0.3 * 2) {
|
||||
this.onTurn?.({ user: userId, transcript: "", reply: "", note: "너무 짧음(<300ms)" });
|
||||
return;
|
||||
}
|
||||
const wav = pcm16MonoToWav(mono, DISCORD_RATE);
|
||||
|
||||
try {
|
||||
@@ -133,9 +146,16 @@ export class VoiceSession {
|
||||
// so the first sentence starts playing while the rest are still spoken.
|
||||
await converseStream(wav, this.getBroadcasting?.(), {
|
||||
onMeta: async (meta) => {
|
||||
if (meta.transcript) {
|
||||
this.onTurn?.({ user: userId, transcript: meta.transcript, reply: meta.reply });
|
||||
}
|
||||
// Report EVERY turn (even empty/VAD-dropped) so the transcript channel
|
||||
// explains why a turn did or didn't answer.
|
||||
this.onTurn?.({
|
||||
user: userId,
|
||||
transcript: meta.transcript,
|
||||
reply: meta.reply,
|
||||
note: meta.note,
|
||||
sttSec: meta.stt_sec,
|
||||
thinkSec: meta.think_sec,
|
||||
});
|
||||
// Apply any broadcast directive the brain requested (e.g. user said
|
||||
// "방송 켜줘 / 꺼줘") before the reply audio plays. The meta line
|
||||
// always precedes the audio clips, so awaiting here preserves order.
|
||||
|
||||
Reference in New Issue
Block a user