feat: show per-stage timing (듣기/LLM/TTS) in the transcript channel

The transcript channel only showed STT and LLM seconds. Add wall-clock
start/end times and durations for listening, LLM and TTS so it's obvious
what takes long; STT surfaces as the gap between listening end and LLM start.

- bridge: emit llm_start_ms/llm_end_ms on meta and tts_*_ms on the end event
- bot: capture the listening window, assemble full timing after the stream,
  and render a per-stage breakdown in the transcript message

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-06-13 23:58:49 +09:00
parent ddebdd7542
commit de5384d166
6 changed files with 242 additions and 34 deletions

View File

@@ -77,8 +77,16 @@ export class VoiceSession {
transcript: string;
reply: string;
note?: string;
sttSec?: number;
thinkSec?: number;
/** Wall-clock epoch-ms markers for each pipeline stage, so the transcript
* channel can show what took long. Listening is measured here (capture
* start -> end of speech); LLM/TTS come from the brain bridge. STT shows
* up as the gap between listenEndMs and llmStartMs. */
listenStartMs?: number;
listenEndMs?: number;
llmStartMs?: number;
llmEndMs?: number;
ttsStartMs?: number;
ttsEndMs?: number;
}) => void;
/** Live screen-share state, sent with each turn so the brain routes search
* (Chrome while broadcasting, Gemini when off). */
@@ -150,6 +158,8 @@ export class VoiceSession {
}
private async captureUtterance(userId: string): Promise<void> {
// "듣기 시작": the moment we begin capturing this speaker's utterance.
const listenStartMs = Date.now();
const opusStream = this.connection.receiver.subscribe(userId, {
end: { behavior: EndBehaviorType.AfterSilence, duration: config.silenceMs },
});
@@ -163,13 +173,23 @@ export class VoiceSession {
pcmStream.on("data", (c: Buffer) => chunks.push(c));
await new Promise<void>((resolve) => pcmStream.once("end", () => resolve()));
// "듣기 종료": end of speech (silence detected). Anything after this is
// STT + reply + TTS on the brain side.
const listenEndMs = Date.now();
if (!chunks.length) return;
const mono = stereoToMono(Buffer.concat(chunks));
// Ignore blips shorter than ~300ms (likely noise / key clicks) — but still
// report them so the transcript channel shows every captured utterance.
if (mono.length < DISCORD_RATE * 0.3 * 2) {
this.onTurn?.({ user: userId, transcript: "", reply: "", note: "너무 짧음(<300ms)" });
this.onTurn?.({
user: userId,
transcript: "",
reply: "",
note: "너무 짧음(<300ms)",
listenStartMs,
listenEndMs,
});
return;
}
const wav = pcm16MonoToWav(mono, DISCORD_RATE);
@@ -178,30 +198,49 @@ export class VoiceSession {
// Streaming turn: the brain sends transcript/reply first, then one audio
// clip per sentence as it is synthesised. We enqueue each clip on arrival
// so the first sentence starts playing while the rest are still spoken.
// The transcript-channel report is sent once the stream ends so it can
// include TTS timing (synthesis runs after the meta line). Audio still
// plays as it arrives — only the diagnostic text post waits.
let metaSeen: {
transcript: string;
reply: string;
note?: string;
llm_start_ms?: number;
llm_end_ms?: number;
} | undefined;
let endSeen: { tts_start_ms?: number; tts_end_ms?: number } | undefined;
await converseStream(wav, this.getBroadcasting?.(), {
onMeta: async (meta) => {
// Report EVERY turn (even empty/VAD-dropped) so the transcript channel
// explains why a turn did or didn't answer.
this.onTurn?.({
user: userId,
transcript: meta.transcript,
reply: meta.reply,
note: meta.note,
sttSec: meta.stt_sec,
thinkSec: meta.think_sec,
});
onMeta: async (m) => {
metaSeen = m;
// Apply any broadcast directive the brain requested (e.g. user said
// "방송 켜줘 / 꺼줘") before the reply audio plays. The meta line
// always precedes the audio clips, so awaiting here preserves order.
if (meta.broadcast_action && this.onBroadcastAction) {
if (m.broadcast_action && this.onBroadcastAction) {
try {
await this.onBroadcastAction(meta.broadcast_action);
await this.onBroadcastAction(m.broadcast_action);
} catch (e) {
console.error("[voice] broadcast action failed:", e);
}
}
},
onAudio: (clip) => this.play(clip),
onEnd: (end) => {
endSeen = end;
},
});
// Report EVERY turn (even empty/VAD-dropped) so the transcript channel
// explains why a turn did or didn't answer, with full stage timing.
this.onTurn?.({
user: userId,
transcript: metaSeen?.transcript ?? "",
reply: metaSeen?.reply ?? "",
note: metaSeen?.note,
listenStartMs,
listenEndMs,
llmStartMs: metaSeen?.llm_start_ms,
llmEndMs: metaSeen?.llm_end_ms,
ttsStartMs: endSeen?.tts_start_ms,
ttsEndMs: endSeen?.tts_end_ms,
});
} catch (err) {
console.error("[voice] converse failed:", err);