From 9f3a57d8a07c788fc8deb003ea3916797b97119c Mon Sep 17 00:00:00 2001 From: EJClaw Date: Tue, 11 Aug 2026 20:51:24 +0900 Subject: [PATCH] feat(discord): M1 persistent selfbot voice joiner + record confirmed build decisions join.mjs joins the target voice channel over the proven DAVE handshake and stays connected, mapping SPEAKING->ssrc and tallying incoming RTP (foundation for M2 audio decrypt). README records GPU=on, shared OAuth brain, natural-but-<=1s TTS, Discord-voice STT input, and the M1..M6 milestones. Co-Authored-By: Claude Opus 4.7 --- README.md | 31 ++++--- dave/join.mjs | 233 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 253 insertions(+), 11 deletions(-) create mode 100644 dave/join.mjs diff --git a/README.md b/README.md index 1210771..f807250 100644 --- a/README.md +++ b/README.md @@ -175,16 +175,25 @@ ffmpeg 디코드 → 프레임. DAVE는 `@snazzah/davey`(Rust NAPI), 수신은 ` --- -## 8. 결정이 필요한 것 (사용자 확인 대기) +## 8. 확정된 결정 (2026-08-11) + 배포 목표 -실제 STT/TTS 엔진을 붙이기 전에 정해야 할 것들: +- **GPU 사용** — OK. RTX 5050(.9)으로 STT/TTS를 돌린다. .9 로컬 도커는 NVIDIA CDI로 GPU 사용 가능. +- **파이썬 버전** — STT/TTS는 3.11(faster-whisper·TTS 자산 호환), 오케스트레이터는 기존 코드 유지. +- **TTS 엔진** — 최대한 사람 같은 음성으로 하되, **전체 지연 1초 초과 금지**가 상한. 후보(XTTS 등)의 + 첫 소리 지연을 GPU에서 실측해, 1초 예산에 맞는 가장 자연스러운 엔진을 채택한다(측정 기반 선택). +- **두뇌** — Claude **OAuth**, 이 호스트에 연결된 것과 **동일 크리덴셜 공유** + (`/home/claude/EJClaw/data/claude/.credentials.json`). API키 아님. +- **STT 입력 경로** — 디스코드 보이스 수신(셀프봇). 유저 음성 Opus RTP를 받아 DAVE 복호·디코드해 STT로. -- **GPU 사용 동의** — RTX 5050으로 STT/TTS를 돌리면 빠르다(규칙상 GPU는 먼저 확인 필요). -- **파이썬 버전** — 이 venv는 3.14인데 faster-whisper·MeloTTS는 3.11대가 안전하다. - 호스트에 이미 MeloTTS용 3.11 자산이 있어, 별도 3.11 venv + subprocess 연결을 제안. -- **엔진 선택** — STT는 faster-whisper, TTS는 한국어 고정. 자연스러움 최우선이면 무거운 - 엔진(XTTS 계열)이 필요하지만 1초 예산과 상충 → 우선 빠른 MeloTTS로 시작할지, 처음부터 - 더 사람 같은 엔진을 GPU로 감수할지 결정 필요. -- **두뇌** — Claude OAuth(Haiku 계열)로 연결(지금 코드는 API키 기반이라 OAuth 전환 필요). -- **STT 입력 경로** — 디스코드 보이스 수신(셀프봇)으로 확정. 유저 음성 Opus RTP를 받아 - 복호·디코드해 STT에 넣는다(로컬 마이크 아님). +### 배포/테스트 목표 +- 대상: 디스코드 서버 `1352269198297923648`의 보이스 채널 `1352269198914621465`. +- 유저봇(셀프봇) 토큰: `.env`의 `DISCORD_SELFBOT_TOKEN`(버너 계정, 밴 리스크 수용). +- .9 로컬 GPU 도커 이미지로 올려, 그 채널에 접속한 뒤 사람이 말하면 대화하도록 한다. +- 첫 로딩 워밍업: 시작 시 모델 프리로드 + 더미 추론(CUDA 워밍) + 보이스 미리 접속 → 첫 대화도 지연 최소. + +### 구현 마일스톤 +- **M1** 셀프봇이 대상 보이스 채널에 상주 접속(DAVE 통과) + 발화자(SSRC) 감지 ← 지금 +- **M2** 유저 음성 Opus RTP 수신 → DAVE 복호 → PCM (라이브 발화자 필요, 최고 난이도) +- **M3** faster-whisper STT(부분전사+VAD) → **M4** Claude OAuth(Haiku) 두뇌 +- **M5** 한국어 TTS 첫 구절 청크를 보이스로 송신(DAVE 암호화) + barge-in +- **M6** 통합 + 워밍업 + .9 GPU 도커 이미지화, 채널 라이브 테스트 diff --git a/dave/join.mjs b/dave/join.mjs new file mode 100644 index 0000000..566748f --- /dev/null +++ b/dave/join.mjs @@ -0,0 +1,233 @@ +// M1 — persistent selfbot voice join (builds on the proven gate.mjs handshake). +// +// Difference from gate.mjs: this does NOT leave after collecting evidence. It +// joins the target voice channel, completes the DAVE/MLS E2EE membership, then +// STAYS connected and reports: +// * who is speaking (op5 SPEAKING -> maps audio SSRC to a user id) +// * incoming UDP/RTP packets per SSRC (the media path the STT stage will read) +// +// This is the foundation for M2 (decrypt the incoming Opus and feed STT). No +// audio is transmitted yet. +// +// Usage: +// node join.mjs # join and stay until killed +// RUN_MS=15000 node join.mjs # join, hold 15s, then leave (for verification) + +import WebSocket from 'ws'; +import dgram from 'node:dgram'; +import fs from 'node:fs'; +import * as davey from '@snazzah/davey'; + +const env = Object.fromEntries( + fs.readFileSync(new URL('../.env', import.meta.url), 'utf8') + .split('\n').filter(l => l && !l.startsWith('#') && l.includes('=')) + .map(l => { const i = l.indexOf('='); return [l.slice(0, i).trim(), l.slice(i + 1).trim()]; }) +); +const TOKEN = env.DISCORD_SELFBOT_TOKEN; +const GUILD_ID = process.env.GUILD_ID || '1352269198297923648'; +const CHANNEL_ID = process.env.CHANNEL_ID || '1352269198914621465'; +const SELF_ID = process.env.SELF_ID || '1513862586112671786'; +const DAVE_VER = process.env.DAVE_VER != null ? Number(process.env.DAVE_VER) : davey.DAVE_PROTOCOL_VERSION; +const RUN_MS = process.env.RUN_MS != null ? Number(process.env.RUN_MS) : 0; // 0 = stay forever + +if (!TOKEN) { console.error('no DISCORD_SELFBOT_TOKEN in .env'); process.exit(2); } + +const t0 = Date.now(); +const log = (...a) => console.log(`[+${String(Date.now() - t0).padStart(6)}ms]`, ...a); +console.log(`davey VERSION=${davey.VERSION} DAVE_PROTOCOL_VERSION=${davey.DAVE_PROTOCOL_VERSION}`); +log(`join start: channel=${CHANNEL_ID} guild=${GUILD_ID} dave=${DAVE_VER}`); + +let mainWs, voiceWs, udp; +let daveSession = null; +let mlsReady = false; +let discoverySelected = false; +const knownUsers = new Set([SELF_ID]); +const ssrcToUser = new Map(); // audio ssrc -> user id +const rtpCount = new Map(); // ssrc -> packet count +const voiceState = { session_id: null, token: null, endpoint: null, ssrc: null, ip: null, port: null, mode: null }; + +function leaveAndExit(code = 0) { + try { mainWs?.send(JSON.stringify({ op: 4, d: { guild_id: GUILD_ID, channel_id: null, self_mute: true, self_deaf: true } })); } catch {} + setTimeout(() => { + try { voiceWs?.close(); } catch {} + try { udp?.close(); } catch {} + try { mainWs?.close(); } catch {} + log(`leaving. speakers seen: ${JSON.stringify([...ssrcToUser.entries()])}, rtp counts: ${JSON.stringify([...rtpCount.entries()])}`); + process.exit(code); + }, 400); +} +process.on('SIGINT', () => { log('SIGINT'); leaveAndExit(0); }); +process.on('SIGTERM', () => { log('SIGTERM'); leaveAndExit(0); }); + +// ---------- MAIN GATEWAY ---------- +mainWs = new WebSocket('wss://gateway.discord.gg/?v=10&encoding=json'); +let mainHb; +mainWs.on('open', () => log('main gw: open')); +mainWs.on('message', (raw) => { + const p = JSON.parse(raw.toString()); + if (p.op === 10) { + mainHb = setInterval(() => { try { mainWs.send(JSON.stringify({ op: 1, d: null })); } catch {} }, p.d.heartbeat_interval); + mainWs.send(JSON.stringify({ op: 2, d: { + token: TOKEN, + capabilities: 16381, + properties: { os: 'Linux', browser: 'Chrome', device: '', system_locale: 'en-US', browser_user_agent: 'Mozilla/5.0', browser_version: '124.0', os_version: '', release_channel: 'stable', client_build_number: 300000 }, + compress: false, + presence: { status: 'invisible', since: 0, activities: [], afk: false }, + }})); + log('main gw: sent IDENTIFY'); + } else if (p.op === 0) { + if (p.t === 'READY') { + log(`main gw: READY as ${p.d.user?.username} (${p.d.user?.id})`); + mainWs.send(JSON.stringify({ op: 4, d: { guild_id: GUILD_ID, channel_id: CHANNEL_ID, self_mute: true, self_deaf: false } })); + log('main gw: sent Voice State Update (join, deaf=false so we can hear)'); + } else if (p.t === 'VOICE_STATE_UPDATE' && p.d.user_id === SELF_ID && p.d.session_id) { + voiceState.session_id = p.d.session_id; log('VOICE_STATE_UPDATE session_id acquired'); maybeConnectVoice(); + } else if (p.t === 'VOICE_SERVER_UPDATE') { + voiceState.token = p.d.token; voiceState.endpoint = p.d.endpoint; + log(`VOICE_SERVER_UPDATE endpoint=${p.d.endpoint}`); maybeConnectVoice(); + } + } +}); +mainWs.on('close', (c, r) => { log(`main gw: close ${c} ${r}`); clearInterval(mainHb); }); +mainWs.on('error', (e) => log('main gw error', e.message)); + +// ---------- VOICE GATEWAY ---------- +function maybeConnectVoice() { + if (voiceWs || !voiceState.session_id || !voiceState.token || !voiceState.endpoint) return; + const url = `wss://${voiceState.endpoint}/?v=8`; + log(`voice gw: connecting ${url}`); + voiceWs = new WebSocket(url); + let voiceHb, lastSeq = null; + + voiceWs.on('open', () => { + voiceWs.send(JSON.stringify({ op: 0, d: { + server_id: GUILD_ID, user_id: SELF_ID, session_id: voiceState.session_id, token: voiceState.token, + max_dave_protocol_version: DAVE_VER, + }})); + log(`voice gw: sent IDENTIFY (max_dave_protocol_version=${DAVE_VER})`); + }); + voiceWs.on('message', (raw, isBinary) => { + if (isBinary) { + const buf = Buffer.isBuffer(raw) ? raw : Buffer.from(raw); + const seq = buf.readUInt16BE(0), op = buf.readUInt8(2), payload = buf.subarray(3); + lastSeq = seq; onDaveBinary(op, payload, seq); + return; + } + handleVoiceJson(JSON.parse(raw.toString())); + }); + voiceWs.on('close', (c, r) => { + log(`voice gw: CLOSE code=${c} reason="${r?.toString() || ''}"`); + clearInterval(voiceHb); + if (c === 4017) { log('FATAL: close 4017 (DAVE rejected)'); leaveAndExit(1); } + }); + voiceWs.on('error', (e) => log('voice gw error', e.message)); + + function handleVoiceJson(p) { + switch (p.op) { + case 8: + voiceHb = setInterval(() => { try { voiceWs.send(JSON.stringify({ op: 3, d: { t: Date.now(), seq_ack: lastSeq ?? 0 } })); } catch {} }, p.d.heartbeat_interval); + log(`voice gw: HELLO hb=${p.d.heartbeat_interval}`); + break; + case 2: // READY + voiceState.ssrc = p.d.ssrc; voiceState.ip = p.d.ip; voiceState.port = p.d.port; + log(`voice gw: READY ssrc=${p.d.ssrc} udp=${p.d.ip}:${p.d.port} modes=${JSON.stringify(p.d.modes)}`); + doUdpDiscoveryAndSelect(p.d); + break; + case 4: // SESSION_DESCRIPTION + voiceState.daveVer = p.d?.dave_protocol_version ?? null; + log(`voice gw: SESSION_DESCRIPTION dave=${voiceState.daveVer} mode=${p.d?.mode}`); + announceReady(); + break; + case 5: // SPEAKING — maps a user to their audio ssrc + if (p.d?.user_id && p.d?.ssrc != null) { + ssrcToUser.set(p.d.ssrc, p.d.user_id); + knownUsers.add(p.d.user_id); + log(`voice gw: SPEAKING user=${p.d.user_id} ssrc=${p.d.ssrc} flags=${p.d.speaking}`); + } + break; + case 11: for (const u of (p.d.user_ids || [])) knownUsers.add(u); log(`voice gw: op11 clients_connect ${JSON.stringify(p.d.user_ids)}`); break; + case 13: if (p.d?.user_id) { knownUsers.delete(p.d.user_id); log(`voice gw: op13 client_disconnect ${p.d.user_id}`); } break; + case 20: if (p.d.user_id) knownUsers.add(p.d.user_id); break; + case 21: try { voiceWs.send(JSON.stringify({ op: 23, d: { transition_id: p.d.transition_id } })); } catch {} log('voice gw: DAVE op21 prepare_transition (acked)'); break; + case 22: log('voice gw: DAVE op22 execute_transition'); break; + default: /* quiet */ break; + } + } + + function onDaveBinary(op, payload) { + try { + switch (op) { + case 25: { + if (!daveSession) daveSession = new davey.DAVESession(voiceState.daveVer || DAVE_VER, SELF_ID, CHANNEL_ID); + daveSession.setExternalSender(payload); + const kp = daveSession.getSerializedKeyPackage(); + voiceWs.send(Buffer.concat([Buffer.from([26]), kp]), { binary: true }); + log(`voice gw: sent DAVE op26 key_package len=${kp.length}`); + break; + } + case 27: { + const res = daveSession.processProposals(payload.readUInt8(0), payload.subarray(1), Array.from(knownUsers)); + if (res && res.commit) { + const parts = [Buffer.from([28]), res.commit]; + if (res.welcome) parts.push(res.welcome); + voiceWs.send(Buffer.concat(parts), { binary: true }); + log(`voice gw: sent DAVE op28 commit_welcome`); + } + break; + } + case 29: daveSession.processCommit(payload.subarray(2)); mlsReady = daveSession.ready; log(`voice gw: op29 commit -> ready=${mlsReady}`); announceReady(); break; + case 30: daveSession.processWelcome(payload.subarray(2)); mlsReady = daveSession.ready; log(`voice gw: op30 welcome -> ready=${mlsReady}`); announceReady(); break; + default: break; + } + } catch (e) { log(`voice gw: DAVE op${op} error: ${e.message}`); } + } + + let announced = false; + function announceReady() { + if (announced) return; + const daveOk = voiceState.daveVer === 0 || mlsReady; + if (voiceState.ssrc != null && voiceState.daveVer != null && daveOk && discoverySelected) { + announced = true; + log(`✅ JOINED & READY. channel=${CHANNEL_ID} dave=${voiceState.daveVer} mlsReady=${mlsReady} privacyCode=${daveSession?.voicePrivacyCode || 'n/a'}`); + log(' staying connected, listening for speakers…'); + if (RUN_MS > 0) setTimeout(() => { log(`RUN_MS=${RUN_MS} elapsed`); leaveAndExit(0); }, RUN_MS); + } + } + + function doUdpDiscoveryAndSelect(ready) { + udp = dgram.createSocket('udp4'); + const disc = Buffer.alloc(74); + disc.writeUInt16BE(1, 0); disc.writeUInt16BE(70, 2); disc.writeUInt32BE(ready.ssrc, 4); + udp.on('message', (msg) => { + if (!discoverySelected) { + discoverySelected = true; + const ipEnd = msg.indexOf(0, 8); + const ip = msg.subarray(8, ipEnd).toString(); + const port = msg.readUInt16BE(msg.length - 2); + const mode = (ready.modes || []).includes('aead_aes256_gcm_rtpsize') ? 'aead_aes256_gcm_rtpsize' + : (ready.modes || []).includes('aead_xchacha20_poly1305_rtpsize') ? 'aead_xchacha20_poly1305_rtpsize' + : (ready.modes || [])[0]; + voiceState.mode = mode; + voiceWs.send(JSON.stringify({ op: 1, d: { protocol: 'udp', data: { address: ip, port, mode }, codecs: [ + { name: 'opus', type: 'audio', priority: 1000, payload_type: 120 }, + ] }})); + log(`voice gw: sent SELECT PROTOCOL (mode=${mode}) after UDP discovery ${ip}:${port}`); + announceReady(); + return; + } + // After selection: these are incoming SRTP media packets. Just tally per-SSRC + // (decryption -> Opus -> PCM is M2). RTP: ssrc at bytes 8..11, pt = byte1 & 0x7f. + if (msg.length >= 12) { + const ssrc = msg.readUInt32BE(8); + rtpCount.set(ssrc, (rtpCount.get(ssrc) || 0) + 1); + const n = rtpCount.get(ssrc); + if (n === 1 || n % 200 === 0) { + const user = ssrcToUser.get(ssrc) || '?'; + log(`rtp: ssrc=${ssrc} user=${user} pt=${msg.readUInt8(1) & 0x7f} count=${n}`); + } + } + }); + udp.on('error', (e) => log('udp error', e.message)); + udp.send(disc, ready.port, ready.ip, (e) => { if (e) log('udp send err', e.message); else log('udp: sent IP discovery'); }); + } +}