Files
game_voice_recorder/app/scripts/stt-eval.mts
EJClaw c32236c3b8 M7: GPU 실사용 시험 (실제 디스코드 음성 28문장, 모델별 인식률·속도)
- voice-e2e: GVR_E2E_SET=demo(28문장, 문장별 CER·지연), GVR_E2E_MODEL/DEVICE
- stt-eval: GVR_EVAL_DEVICE=cuda, 틀린 문장 모두 출력, GVR_EVAL_HELDOUT(학습에 안 쓴 문장 CER)
- bot: GVR_RECORD_BOTS 시험 모드에서 봇만 있는 방을 빈 방으로 보고 30초 뒤 나가던 문제

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 04:21:57 +09:00

81 lines
4.3 KiB
TypeScript

// M7 인식률 평가: 시연 대화 28문장(TTS)을 모델별로 변환해 CER(글자 오류율)을 잰다. 사전(핫워드+교정) 있음/없음 비교.
// node --import tsx scripts/stt-eval.mts <models_dir> <model...>
// GVR_EVAL_HELDOUT=0,3,5 : 파인튜닝 교정에 쓴 문장 번호. 이 문장을 뺀 CER 도 따로 보여 준다
// GPU: GVR_EVAL_DEVICE=cuda (float16). 리눅스는 LD_LIBRARY_PATH 에 cuBLAS·cuDNN 12/9 경로가 있어야 한다.
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { Store } from '../src/core/db/db.ts';
import { Dictionary } from '../src/core/dict/dictionary.ts';
import { WorkerClient } from '../src/core/stt/worker.ts';
const [modelsDir, ...models] = process.argv.slice(2);
const device = process.env.GVR_EVAL_DEVICE === 'cuda' ? 'cuda' : 'cpu';
const computeType = device === 'cuda' ? 'float16' : 'int8';
const seenIdx = new Set((process.env.GVR_EVAL_HELDOUT || '').split(',').filter(Boolean).map(Number));
const root = path.resolve(import.meta.dirname, '..', '..');
const clipDir = path.join(os.homedir(), '.tmp', 'gvr-demo-clips');
const demo = fs.readFileSync(path.join(import.meta.dirname, 'demo-stack.mts'), 'utf8');
const lines = [...demo.matchAll(/\{ u: \d, text: '([^']+)', at: \{ (val|lol|off)/g)].map((m) => ({ text: m[1], ctx: m[2] === 'val' ? 'valorant' : m[2] === 'lol' ? 'lol' : 'none' })) as { text: string; ctx: 'valorant' | 'lol' | 'none' }[];
const norm = (s: string) => s.replace(/[\s.,!?~]/g, '').replace(/PC방/g, '피시방').replace(/2명/g, '두명');
function cer(ref: string, hyp: string): number {
const a = [...norm(ref)];
const b = [...norm(hyp)];
const d = Array.from({ length: a.length + 1 }, (_, i) => [i, ...new Array(b.length).fill(0)]);
for (let j = 1; j <= b.length; j++) d[0][j] = j;
for (let i = 1; i <= a.length; i++) for (let j = 1; j <= b.length; j++) d[i][j] = Math.min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1));
return d[a.length][b.length] / Math.max(1, a.length);
}
const tmp = fs.mkdtempSync(path.join(os.homedir(), '.tmp', 'gvr-eval-'));
const store = new Store(path.join(tmp, 'db.sqlite'));
const dict = new Dictionary(store, path.join(root, 'app', 'resources'), () => ({}));
dict.seed();
const w = new WorkerClient({ cmd: path.join(root, 'worker/.venv/bin/python'), args: ['-m', 'gvr_worker'], cwd: path.join(root, 'worker'), env: { GVR_DATA_DIR: tmp, GVR_MODELS_DIR: modelsDir, GVR_LOG_LEVEL: 'WARNING' } });
w.start();
await new Promise((r) => w.once('ready', r));
const report: string[] = [];
for (const model of models) {
const t0 = Date.now();
await w.request('load', { slot: 'realtime', model, device, compute_type: computeType }, 30 * 60000);
const loadS = (Date.now() - t0) / 1000;
for (const useDict of [false, true]) {
let sum = 0;
let exact = 0;
let ms = 0;
let unseenSum = 0;
let unseenN = 0;
const rows: string[] = [];
for (const [i, l] of lines.entries()) {
const r = await w.request('transcribe', {
slot: 'realtime',
path: path.join(clipDir, `${i}.ogg`),
hotwords: useDict ? dict.hotwords(l.ctx) : '',
corrections: useDict ? dict.corrections() : [],
dictionary: useDict ? dict.candidates(l.ctx) : [],
target_lufs: -18,
noise_sensitivity: 0.5,
denoise: false,
profile_lufs: null,
}, 10 * 60000);
const hyp = r.text ?? '';
const c = cer(l.text, hyp);
sum += c;
if (!seenIdx.has(i)) {
unseenSum += c;
unseenN++;
}
if (c === 0) exact++;
ms += r.elapsed_ms;
rows.push(` ${c === 0 ? '=' : c < 0.15 ? '≈' : '≠'} ${l.text} → ${hyp}`);
}
const line = `${model} ${useDict ? '사전 적용' : '사전 없음'}: CER ${((sum / lines.length) * 100).toFixed(1)}% · 완전 일치 ${exact}/${lines.length} · 평균 ${(ms / lines.length / 1000).toFixed(2)}초/문장 (${device} ${computeType}) · 적재 ${loadS.toFixed(0)}초${seenIdx.size ? ` · 학습에 안 쓴 ${unseenN}문장 CER ${((unseenSum / unseenN) * 100).toFixed(1)}%` : ''}`;
console.log(line);
console.log(rows.filter((r) => !r.startsWith(' =')).join('\n'));
report.push(line);
}
await w.request('unload', { slot: 'realtime' });
}
console.log('\n== 요약 ==\n' + report.join('\n'));
await w.stop();
process.exit(0);