feat: couple broadcast to voice + voice-controlled broadcast toggle
Completes the STREAM_BROWSER=true behaviour:
- handleJoin auto-starts the broadcast on voice join and wires the session to
the guild streamer; each turn reports the live state to the brain so search
routes Chrome (live) vs Gemini (off).
- New setBroadcast tool lets the user toggle the broadcast by voice ("방송
켜줘/꺼줘") via the LLM (no hardcoded phrases); it refuses when
STREAM_BROWSER=false. The directive flows brain -> bridge (broadcast_action)
-> bot streamer.start/stop, guarded by isActive() so it's idempotent.
- Per-turn IPC uses a thread-local (reply/turn_state.py) instead of threading
params through the whole engine chain: bridge sets broadcasting in, tool
records the directive out; Tool.execute exposes broadcasting on ToolContext.
Bot typecheck clean; brain covered by tests/test_set_broadcast.py (+ existing
routing tests). Specs + docs updated.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -4,11 +4,15 @@
|
||||
*/
|
||||
import { config } from "./config.ts";
|
||||
|
||||
export type BroadcastAction = "start" | "stop";
|
||||
|
||||
export interface ConverseResult {
|
||||
transcript: string;
|
||||
language?: string | null;
|
||||
reply: string;
|
||||
error?: string | null;
|
||||
/** Broadcast directive the brain requested this turn (setBroadcast tool). */
|
||||
broadcast_action?: BroadcastAction | null;
|
||||
/** base64-encoded 16-bit PCM WAV of the spoken reply, or null if TTS off */
|
||||
audio_b64?: string | null;
|
||||
}
|
||||
@@ -16,12 +20,16 @@ export interface ConverseResult {
|
||||
export interface TextResult {
|
||||
reply: string;
|
||||
error?: string | null;
|
||||
broadcast_action?: BroadcastAction | null;
|
||||
audio_b64?: string | null;
|
||||
}
|
||||
|
||||
/** Full voice turn: WAV in -> {transcript, reply, reply audio}. */
|
||||
export async function converse(wav: Buffer): Promise<ConverseResult> {
|
||||
const res = await fetch(`${config.bridgeUrl}/converse`, {
|
||||
/** Full voice turn: WAV in -> {transcript, reply, reply audio}. ``broadcasting``
|
||||
* is the current live screen-share state so the brain routes search (Chrome
|
||||
* while live, Gemini when off). */
|
||||
export async function converse(wav: Buffer, broadcasting?: boolean): Promise<ConverseResult> {
|
||||
const qs = broadcasting === undefined ? "" : `?broadcasting=${broadcasting ? "1" : "0"}`;
|
||||
const res = await fetch(`${config.bridgeUrl}/converse${qs}`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "audio/wav" },
|
||||
body: wav,
|
||||
|
||||
@@ -81,6 +81,30 @@ async function handleJoin(i: ChatInputCommandInteraction) {
|
||||
const session = await joinChannel(channel);
|
||||
session.onTurn = ({ transcript, reply }) =>
|
||||
console.log(`🗣️ ${transcript}\n🤖 ${reply}`);
|
||||
|
||||
// Screen-share mode (STREAM_BROWSER=true): the broadcast is coupled to the
|
||||
// voice session. Auto-start it on join, report its live state to the brain
|
||||
// each turn (search routes Chrome while live / Gemini when off), and let the
|
||||
// brain toggle it via voice ("방송 켜줘 / 꺼줘"). In voice-only mode
|
||||
// (STREAM_BROWSER=false) none of this runs and the broadcast stays off.
|
||||
if (config.screenBrowser) {
|
||||
const streamer = await getStreamer(i.guildId!);
|
||||
const ctx: StreamContext = { guildId: i.guildId!, voiceChannelId: channel.id };
|
||||
session.getBroadcasting = () => streamer.isActive();
|
||||
session.onBroadcastAction = async (action) => {
|
||||
if (action === "start") {
|
||||
if (!streamer.isActive()) await streamer.start(ctx);
|
||||
} else if (streamer.isActive()) {
|
||||
await streamer.stop();
|
||||
}
|
||||
};
|
||||
try {
|
||||
if (!streamer.isActive()) await streamer.start(ctx);
|
||||
} catch (e) {
|
||||
console.error("[join] auto-broadcast failed:", e);
|
||||
}
|
||||
}
|
||||
|
||||
await i.editReply(`🎙️ '${channel.name}' 채널에 접속했습니다. 말씀하세요.`);
|
||||
}
|
||||
|
||||
|
||||
@@ -71,6 +71,11 @@ export class VoiceSession {
|
||||
private playQueue: Buffer[] = [];
|
||||
/** Optional callback to surface transcripts/replies to a text channel. */
|
||||
onTurn?: (info: { user: string; transcript: string; reply: string }) => void;
|
||||
/** Live screen-share state, sent with each turn so the brain routes search
|
||||
* (Chrome while broadcasting, Gemini when off). */
|
||||
getBroadcasting?: () => boolean;
|
||||
/** Apply a broadcast directive the brain requested (start/stop the stream). */
|
||||
onBroadcastAction?: (action: "start" | "stop") => void | Promise<void>;
|
||||
|
||||
constructor(channel: VoiceBasedChannel) {
|
||||
this.guildId = channel.guild.id;
|
||||
@@ -123,10 +128,19 @@ export class VoiceSession {
|
||||
const wav = pcm16MonoToWav(mono, DISCORD_RATE);
|
||||
|
||||
try {
|
||||
const result = await converse(wav);
|
||||
const result = await converse(wav, this.getBroadcasting?.());
|
||||
if (result.transcript) {
|
||||
this.onTurn?.({ user: userId, transcript: result.transcript, reply: result.reply });
|
||||
}
|
||||
// Apply any broadcast directive the brain requested (e.g. user said
|
||||
// "방송 켜줘 / 꺼줘") before playing the reply.
|
||||
if (result.broadcast_action && this.onBroadcastAction) {
|
||||
try {
|
||||
await this.onBroadcastAction(result.broadcast_action);
|
||||
} catch (e) {
|
||||
console.error("[voice] broadcast action failed:", e);
|
||||
}
|
||||
}
|
||||
const audio = decodeWav(result.audio_b64);
|
||||
if (audio) this.play(audio);
|
||||
} catch (err) {
|
||||
|
||||
Reference in New Issue
Block a user