diff --git a/bot/bun.lock b/bot/bun.lock index 7705ed7..2cceafb 100644 --- a/bot/bun.lock +++ b/bot/bun.lock @@ -5,7 +5,9 @@ "": { "name": "javis-bot", "dependencies": { - "@discordjs/voice": "^0.18.0", + "@discordjs/voice": "^0.19.2", + "@snazzah/davey": "^0.1.11", + "debug": "^4.3.7", "discord.js": "^14.16.3", "dotenv": "^16.4.5", "libsodium-wrappers": "^0.7.15", @@ -45,7 +47,7 @@ "@discordjs/util": ["@discordjs/util@1.2.0", "", { "dependencies": { "discord-api-types": "^0.38.33" } }, "sha512-3LKP7F2+atl9vJFhaBjn4nOaSWahZ/yWjOvA4e5pnXkt2qyXRCHLxoBQy81GFtLGCq7K9lPm9R517M1U+/90Qg=="], - "@discordjs/voice": ["@discordjs/voice@0.18.0", "", { "dependencies": { "@types/ws": "^8.5.12", "discord-api-types": "^0.37.103", "prism-media": "^1.3.5", "tslib": "^2.6.3", "ws": "^8.18.0" } }, "sha512-BvX6+VJE5/vhD9azV9vrZEt9hL1G+GlOdsQaVl5iv9n87fkXjf3cSwllhR3GdaUC8m6dqT8umXIWtn3yCu4afg=="], + "@discordjs/voice": ["@discordjs/voice@0.19.2", "", { "dependencies": { "@snazzah/davey": "^0.1.9", "@types/ws": "^8.18.1", "discord-api-types": "^0.38.41", "prism-media": "^1.3.5", "tslib": "^2.8.1", "ws": "^8.19.0" } }, "sha512-3yJ255e4ag3wfZu/DSxeOZK1UtnqNxnspmLaQetGT0pDkThNZoHs+Zg6dgZZ19JEVomXygvfHn9lNpICZuYtEA=="], "@discordjs/ws": ["@discordjs/ws@1.2.3", "", { "dependencies": { "@discordjs/collection": "^2.1.0", "@discordjs/rest": "^2.5.1", "@discordjs/util": "^1.1.0", "@sapphire/async-queue": "^1.5.2", "@types/ws": "^8.5.10", "@vladfrangu/async_event_emitter": "^2.2.4", "discord-api-types": "^0.38.1", "tslib": "^2.6.2", "ws": "^8.17.0" } }, "sha512-wPlQDxEmlDg5IxhJPuxXr3Vy9AjYq5xCvFWGJyD7w7Np8ZGu+Mc+97LCoEc/+AYCo2IDpKioiH0/c/mj5ZR9Uw=="], @@ -541,8 +543,6 @@ "@discordjs/rest/undici": ["undici@6.24.1", "", {}, "sha512-sC+b0tB1whOCzbtlx20fx3WgCXwkW627p4EA9uM+/tNNPkSS+eSEld6pAs9nDv7WbY1UUljBMYPtu9BCOrCWKA=="], - "@discordjs/voice/discord-api-types": ["discord-api-types@0.37.120", "", {}, "sha512-7xpNK0EiWjjDFp2nAhHXezE4OUWm7s1zhc/UXXN6hnFFU8dfoPHgV0Hx0RPiCa3ILRpdeh152icc68DGCyXYIw=="], - "bl/buffer": ["buffer@5.7.1", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" } }, "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ=="], "bl/readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], diff --git a/bot/package.json b/bot/package.json index 44b00c8..426a6bb 100644 --- a/bot/package.json +++ b/bot/package.json @@ -11,7 +11,9 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@discordjs/voice": "^0.18.0", + "@discordjs/voice": "^0.19.2", + "@snazzah/davey": "^0.1.11", + "debug": "^4.3.7", "discord.js": "^14.16.3", "dotenv": "^16.4.5", "libsodium-wrappers": "^0.7.15", diff --git a/bot/src/audio.ts b/bot/src/audio.ts deleted file mode 100644 index e3986d4..0000000 --- a/bot/src/audio.ts +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Shared PCM/WAV helpers for the voice paths (normal-bot @discordjs/voice and - * the userbot/selfbot receiver). Discord delivers 48 kHz, 2-channel, 16-bit LE - * PCM either way, so the brain-bound WAV is built the same. - */ - -export const DISCORD_RATE = 48000; -export const DISCORD_CHANNELS = 2; - -/** Build a minimal PCM16 mono WAV around raw little-endian samples. */ -export function pcm16MonoToWav(pcm: Buffer, sampleRate: number): Buffer { - const header = Buffer.alloc(44); - const dataLen = pcm.length; - header.write("RIFF", 0); - header.writeUInt32LE(36 + dataLen, 4); - header.write("WAVE", 8); - header.write("fmt ", 12); - header.writeUInt32LE(16, 16); - header.writeUInt16LE(1, 20); // PCM - header.writeUInt16LE(1, 22); // mono - header.writeUInt32LE(sampleRate, 24); - header.writeUInt32LE(sampleRate * 2, 28); // byte rate (mono * 2 bytes) - header.writeUInt16LE(2, 32); // block align - header.writeUInt16LE(16, 34); // bits per sample - header.write("data", 36); - header.writeUInt32LE(dataLen, 40); - return Buffer.concat([header, pcm]); -} - -/** Downmix interleaved stereo PCM16 to mono PCM16. */ -export function stereoToMono(stereo: Buffer): Buffer { - const samples = Math.floor(stereo.length / 4); // 2 ch * 2 bytes - const mono = Buffer.alloc(samples * 2); - for (let i = 0; i < samples; i++) { - const l = stereo.readInt16LE(i * 4); - const r = stereo.readInt16LE(i * 4 + 2); - mono.writeInt16LE((l + r) >> 1, i * 2); - } - return mono; -} diff --git a/bot/src/userbot.ts b/bot/src/userbot.ts index 96edaa2..28c2b46 100644 --- a/bot/src/userbot.ts +++ b/bot/src/userbot.ts @@ -1,29 +1,26 @@ /** - * Userbot (selfbot) entry — runs the whole assistant on a single USER account - * token, which is the only kind of account Discord lets Go Live (screen-share). + * Userbot (selfbot) entry — runs the assistant on a single USER account token, + * which is the only kind of account Discord lets Go Live (screen-share). * - * Stage 1 (this module): log in, join a voice channel (auto from - * DISCORD_VOICE_CHANNEL_ID, or via the "!자비스 join/leave" text command), listen - * to users (selfbot VoiceReceiver -> Whisper via the bridge), and speak replies - * back (TTS -> playAudio). Go-Live broadcast + broadcast-coupled search routing - * are wired in Stage 2 on the SAME voice connection (createStreamConnection). + * Voice goes through @discordjs/voice (>=0.19 + @snazzah/davey for Discord's + * DAVE E2EE) using the selfbot client's `voiceAdapterCreator`. The selfbot + * library's OWN native voice connection does NOT complete the current + * (DAVE/v8) handshake and times out, so we deliberately reuse the normal-bot + * VoiceSession (receive -> Whisper -> reply -> TTS playback) instead, just with + * a user-account channel. + * + * Stage 1 (this module): log in, join (auto from DISCORD_VOICE_CHANNEL_ID or via + * "!자비스 join"), converse by voice. Go-Live broadcast + broadcast-coupled + * search routing land in stage 2. * * ToS WARNING: automating a user account ("selfbot") violates Discord's ToS and * can get the account banned. Use a throwaway/burner account only. */ -import { Readable } from "node:stream"; +import type { VoiceBasedChannel } from "discord.js"; import { config } from "./config.ts"; -import { converse, decodeWav } from "./bridge.ts"; -import { DISCORD_RATE, pcm16MonoToWav, stereoToMono } from "./audio.ts"; +import { joinChannel, leaveGuild } from "./voice.ts"; -// The selfbot lib is an optional native dependency (same as the streamer path), -// loaded dynamically so the normal-bot build runs without it. Its types are -// loose, so the voice surface is treated as `any`. type AnyClient = any; -type AnyConnection = any; - -const listening = new Set(); -let connection: AnyConnection = null; async function loadSelfbot(): Promise { try { @@ -36,73 +33,17 @@ async function loadSelfbot(): Promise { } } -/** Capture one user's utterance, send it to the brain, speak the reply back. */ -async function handleUtterance(userId: string, conn: AnyConnection): Promise { - // PCM mode: the receiver decodes Opus to 48 kHz stereo s16le for us. - const pcmStream: Readable = conn.receiver.createStream(userId, { - mode: "pcm", - end: "silence", - }); - const chunks: Buffer[] = []; - pcmStream.on("data", (c: Buffer) => chunks.push(c)); - await new Promise((resolve) => pcmStream.once("end", () => resolve())); - - if (!chunks.length) return; - const mono = stereoToMono(Buffer.concat(chunks)); - // Ignore blips shorter than ~300ms (likely noise / key clicks). - if (mono.length < DISCORD_RATE * 0.3 * 2) return; - const wav = pcm16MonoToWav(mono, DISCORD_RATE); - - try { - // broadcasting is wired in Stage 2 (Go-Live state); until then it is off so - // real-time search uses the Gemini path. - const result = await converse(wav, false); - if (result.transcript) { - console.log(`🗣️ ${result.transcript}\n🤖 ${result.reply}`); - } - const audio = decodeWav(result.audio_b64); - if (audio) { - try { - conn.playAudio(Readable.from(audio)); - } catch (e) { - console.error("[userbot] playAudio failed:", e); - } - } - } catch (err) { - console.error("[userbot] converse failed:", err); - } -} - async function joinAndListen(client: AnyClient, channelId: string): Promise { const channel: any = await client.channels.fetch(channelId).catch(() => null); - if (!channel || !channel.isVoice?.()) { + if (!channel || channel.isVoice?.() === false) { console.error(`[userbot] voice channel ${channelId} not found / not a voice channel`); return; } - connection = await client.voice.joinChannel(channel, { - selfMute: false, - selfDeaf: false, - selfVideo: false, - }); + // The selfbot VoiceChannel is runtime-compatible with @discordjs/voice's + // joinVoiceChannel (it exposes id, guild.id and guild.voiceAdapterCreator). + const session = await joinChannel(channel as unknown as VoiceBasedChannel); + session.onTurn = ({ transcript, reply }) => console.log(`🗣️ ${transcript}\n🤖 ${reply}`); console.log(`🎙️ 유저봇이 '${channel.name}' 음성채널에 참여했습니다.`); - - connection.on("speaking", (user: any, speaking: any) => { - const uid = user?.id; - if (!uid || !speaking?.bitfield) return; // bitfield 0 = stopped speaking - if (listening.has(uid)) return; - listening.add(uid); - handleUtterance(uid, connection).finally(() => listening.delete(uid)); - }); -} - -function leave(): void { - try { - connection?.disconnect?.(); - } catch { - /* already gone */ - } - connection = null; - listening.clear(); } export async function runUserbot(): Promise { @@ -123,18 +64,15 @@ export async function runUserbot(): Promise { } }); - // Text-command control (userbots cannot use slash commands). The command - // joins the caller's current voice channel, or leaves. + // Text-command control (userbots cannot use slash commands): join the + // caller's current voice channel, or leave. client.on("messageCreate", async (msg: any) => { - if (msg.author?.id !== client.user?.id && !config.autoJoinChannelId) { - // Only the account owner drives it when there's no fixed channel. - } const content = (msg.content || "").trim(); if (content === "!자비스 join" || content === "!jarvis join") { const ch = msg.member?.voice?.channel || msg.author?.voice?.channel; if (ch) await joinAndListen(client, ch.id).catch((e) => console.error("[userbot] join cmd:", e)); } else if (content === "!자비스 leave" || content === "!jarvis leave") { - leave(); + leaveGuild(config.guildId); } });