Files
game_voice_recorder/app/scripts/voice-e2e.mts
EJClaw c32236c3b8 M7: GPU 실사용 시험 (실제 디스코드 음성 28문장, 모델별 인식률·속도)
- voice-e2e: GVR_E2E_SET=demo(28문장, 문장별 CER·지연), GVR_E2E_MODEL/DEVICE
- stt-eval: GVR_EVAL_DEVICE=cuda, 틀린 문장 모두 출력, GVR_EVAL_HELDOUT(학습에 안 쓴 문장 CER)
- bot: GVR_RECORD_BOTS 시험 모드에서 봇만 있는 방을 빈 방으로 보고 30초 뒤 나가던 문제

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 04:21:57 +09:00

134 lines
8.5 KiB
TypeScript

// M1 실제 음성 E2E: 테스트봇이 음성채널에 들어가 한국어 문장을 말하면, 겜녹이(코어)가 따라 들어와 녹음 → STT → DB 저장까지.
// node --import tsx scripts/voice-e2e.mts <guildId> <voiceChannelId>
// 필요: secrets.json 의 겜녹이·테스트봇 토큰, worker/.venv, tiny 모델(HF 캐시)
// M7 실사용 시험: GVR_E2E_SET=demo (시연 대화 28문장, 문장별 CER·지연), GVR_E2E_MODEL=<모델 폴더 경로>, GVR_E2E_DEVICE=cuda
import { Client, GatewayIntentBits } from 'discord.js';
import { joinVoiceChannel, entersState, VoiceConnectionStatus, createAudioPlayer, createAudioResource, StreamType, AudioPlayerStatus } from '@discordjs/voice';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { synthesize } from '../src/core/discord/tts.ts';
const [guildId, channelId] = process.argv.slice(2);
const secrets = JSON.parse(fs.readFileSync('/home/claude/.config/ejclaw/secrets.json', 'utf8')).credentials;
const GVR_TOKEN = secrets['discord.com/bot/game-voice-recorder'].token;
const SPEAKER_TOKEN = secrets['discord.com/bot/testbot'].token;
const SPEAKER_ID = secrets['discord.com/bot/testbot'].client_id || '1538122882528321536';
const root = path.resolve(import.meta.dirname, '..', '..');
const work = fs.mkdtempSync(path.join(process.env.TMPDIR || os.tmpdir(), 'gvr-voice-e2e-'));
const models = path.join(work, 'models');
fs.mkdirSync(models, { recursive: true });
const hf = path.join(os.homedir(), '.cache/huggingface/hub/models--Systran--faster-whisper-small/snapshots');
const modelArg = process.env.GVR_E2E_MODEL;
const model = modelArg ? path.basename(modelArg) : fs.existsSync(hf) ? 'small' : 'tiny';
const device = process.env.GVR_E2E_DEVICE === 'cuda' ? 'cuda' : 'cpu';
fs.symlinkSync(modelArg ? path.resolve(modelArg) : path.join(hf.replace('small', model), fs.readdirSync(hf.replace('small', model))[0]), path.join(models, model));
fs.writeFileSync(
path.join(work, 'settings.json'),
JSON.stringify({
discord: { guildId, followUserId: SPEAKER_ID, followEnabled: true, ttsEnabled: true, ttsOnNewMember: false },
stt: { realtimeModel: model, preciseModel: model, device, computeType: device === 'cuda' ? 'float16' : 'int8' },
games: { valorant: false, lol: false },
}),
);
process.env.GVR_RECORD_BOTS = '1';
process.env.GVR_BOT_TOKEN = GVR_TOKEN;
const DEMO = process.env.GVR_E2E_SET === 'demo';
const demoSrc = fs.readFileSync(path.join(import.meta.dirname, 'demo-stack.mts'), 'utf8');
const SENTENCES = DEMO
? [...demoSrc.matchAll(/\{ u: \d, text: '([^']+)', at: \{ (?:val|lol|off)/g)].map((m) => m[1])
: ['레이즈 궁 빠졌어 B 사이트 로테 하자', '오늘 저녁에 치킨 먹을래', '나 오퍼레이터 살게'];
const clips: string[] = [];
const clipDir = path.join(os.homedir(), '.tmp', DEMO ? 'gvr-demo-clips' : 'gvr-e2e-clips');
fs.mkdirSync(clipDir, { recursive: true });
for (const [i, s] of SENTENCES.entries()) {
const f = path.join(clipDir, DEMO ? `${i}.webm` : `say-${i}.webm`);
if (!fs.existsSync(f)) await synthesize(s, i % 2 ? 'ko-KR-InJoonNeural' : 'ko-KR-SunHiNeural', f);
clips.push(f);
}
const { startCore } = await import('../src/core/index.ts');
const core = await startCore({ dataDir: work, resourcesDir: path.join(root, 'app', 'resources'), worker: { cmd: path.join(root, 'worker/.venv/bin/python'), args: ['-m', 'gvr_worker'], cwd: path.join(root, 'worker') }, version: 'e2e' });
const wait = async (fn: () => boolean, ms: number, label: string) => {
const t = Date.now();
while (!fn()) {
if (Date.now() - t > ms) throw new Error('timeout: ' + label);
await new Promise((r) => setTimeout(r, 200));
}
};
let ok = false;
const spk = new Client({ intents: [GatewayIntentBits.Guilds, GatewayIntentBits.GuildVoiceStates] });
try {
await wait(() => core.bot.state.status === 'ready', 30000, 'gvr bot ready');
await wait(() => core.stt.status.slots.realtime.state === 'loaded', 120000, 'model');
await spk.login(SPEAKER_TOKEN);
await new Promise((r) => spk.once('clientReady', r));
const ch: any = await spk.channels.fetch(channelId);
const conn = joinVoiceChannel({ channelId, guildId, adapterCreator: ch.guild.voiceAdapterCreator, selfDeaf: false, selfMute: false, daveEncryption: true });
await entersState(conn, VoiceConnectionStatus.Ready, 20000);
console.log('speaker joined', ch.name);
await wait(() => core.bot.state.recording.active, 20000, 'gvr follows speaker');
console.log('겜녹이 따라 들어옴 (follow):', core.bot.state.recording.channel);
await new Promise((r) => setTimeout(r, 9000)); // 안내 음성이 끝나고 DAVE 키 교환이 안정될 때까지
// 발화가 끝난 뒤 글자가 나오기까지 걸린 시간 (실시간 모델 지연)
const seen = new Map<string, number>();
const poll = setInterval(() => {
for (const r of core.store.db.prepare(`SELECT id FROM utterances WHERE status!='pending'`).all() as { id: string }[]) if (!seen.has(r.id)) seen.set(r.id, Date.now());
}, 100);
const player = createAudioPlayer();
conn.subscribe(player);
for (const f of clips) {
player.play(createAudioResource(fs.createReadStream(f), { inputType: StreamType.WebmOpus }));
await entersState(player, AudioPlayerStatus.Playing, 5000);
await entersState(player, AudioPlayerStatus.Idle, 30000);
await new Promise((r) => setTimeout(r, 1500));
}
await wait(() => core.store.db.prepare(`SELECT count(*) n FROM utterances WHERE status!='pending'`).get().n >= SENTENCES.length, 90000, 'stt');
clearInterval(poll);
const rows = core.store.db.prepare('SELECT id, user_id, started_at, ended_at, text, gain_db, status FROM utterances ORDER BY started_at').all() as any[];
const lat = rows.map((r) => (seen.get(r.id) ?? Date.now()) - r.ended_at);
for (const [i, r] of rows.entries()) console.log(` [${r.user_id === SPEAKER_ID ? '테스트봇' : r.user_id}] ${((r.ended_at - r.started_at) / 1000).toFixed(1)}s gain ${r.gain_db?.toFixed(1)}dB 지연 ${(lat[i] / 1000).toFixed(2)}s ${r.status}: ${r.text}`);
const sorted = [...lat].sort((a, b) => a - b);
console.log(`발화 ${rows.length}개 (말한 문장 ${SENTENCES.length}개) · 모델 ${model} (${device}) · 지연 중앙값 ${(sorted[Math.floor(sorted.length / 2)] / 1000).toFixed(2)}s · 최대 ${(sorted[sorted.length - 1] / 1000).toFixed(2)}s`);
let cerAvg = 0;
if (DEMO) {
// 발화가 문장 수와 같으면 순서대로 짝지어 CER 을 잰다 (stt-eval 과 같은 정규화)
const norm = (x: string) => x.replace(/[\s.,!?~]/g, '').replace(/PC방/g, '피시방').replace(/2명/g, '두명');
const cer = (a0: string, b0: string) => {
const a = [...norm(a0)];
const b = [...norm(b0)];
const d = Array.from({ length: a.length + 1 }, (_, i) => [i, ...new Array(b.length).fill(0)]);
for (let j = 1; j <= b.length; j++) d[0][j] = j;
for (let i = 1; i <= a.length; i++) for (let j = 1; j <= b.length; j++) d[i][j] = Math.min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1));
return d[a.length][b.length] / Math.max(1, a.length);
};
if (rows.length === SENTENCES.length) {
const cs = rows.map((r, i) => cer(SENTENCES[i], r.text || ''));
cerAvg = cs.reduce((a, b) => a + b, 0) / cs.length;
console.log(`CER ${(cerAvg * 100).toFixed(1)}% · 완전 일치 ${cs.filter((c) => c === 0).length}/${cs.length}`);
} else {
cerAvg = 1;
console.log('발화 수가 문장 수와 달라 CER 을 짝지을 수 없음 (발화 분할 확인 필요)');
}
}
// 녹음 제외 확인
core.store.setExcluded(SPEAKER_ID, true);
const before = rows.length;
player.play(createAudioResource(fs.createReadStream(clips[0]), { inputType: StreamType.WebmOpus }));
await entersState(player, AudioPlayerStatus.Idle, 30000).catch(() => undefined);
await new Promise((r) => setTimeout(r, 2500));
const after = (core.store.db.prepare('SELECT count(*) n FROM utterances').get() as any).n;
console.log(`녹음 제외 후 새 발화: ${after - before}개 (0 이어야 함)`);
const texts = rows.map((r) => r.text || '').join(' ');
ok = rows.length >= SENTENCES.length && /레이즈|오퍼|치킨/.test(texts) && after === before && (!DEMO || cerAvg < 0.15);
conn.destroy();
await wait(() => !core.bot.state.recording.active, 20000, 'gvr leaves after speaker').catch(() => console.log('(겜녹이 퇴장 대기 시간 초과)'));
console.log('겜녹이 퇴장:', !core.bot.state.recording.active);
} catch (e) {
console.error('E2E 실패', e);
} finally {
await spk.destroy();
await core.stop();
console.log(ok ? 'VOICE E2E PASS' : 'VOICE E2E FAIL');
process.exit(ok ? 0 : 1);
}