// M7 인식률 평가: 시연 대화 28문장(TTS)을 모델별로 변환해 CER(글자 오류율)을 잰다. 사전(핫워드+교정) 있음/없음 비교. // node --import tsx scripts/stt-eval.mts // GVR_EVAL_HELDOUT=0,3,5 : 파인튜닝 교정에 쓴 문장 번호. 이 문장을 뺀 CER 도 따로 보여 준다 // GPU: GVR_EVAL_DEVICE=cuda (float16). 리눅스는 LD_LIBRARY_PATH 에 cuBLAS·cuDNN 12/9 경로가 있어야 한다. import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { Store } from '../src/core/db/db.ts'; import { Dictionary } from '../src/core/dict/dictionary.ts'; import { WorkerClient } from '../src/core/stt/worker.ts'; const [modelsDir, ...models] = process.argv.slice(2); const device = process.env.GVR_EVAL_DEVICE === 'cuda' ? 'cuda' : 'cpu'; const computeType = device === 'cuda' ? 'float16' : 'int8'; const seenIdx = new Set((process.env.GVR_EVAL_HELDOUT || '').split(',').filter(Boolean).map(Number)); const root = path.resolve(import.meta.dirname, '..', '..'); const clipDir = path.join(os.homedir(), '.tmp', 'gvr-demo-clips'); const demo = fs.readFileSync(path.join(import.meta.dirname, 'demo-stack.mts'), 'utf8'); const lines = [...demo.matchAll(/\{ u: \d, text: '([^']+)', at: \{ (val|lol|off)/g)].map((m) => ({ text: m[1], ctx: m[2] === 'val' ? 'valorant' : m[2] === 'lol' ? 'lol' : 'none' })) as { text: string; ctx: 'valorant' | 'lol' | 'none' }[]; const norm = (s: string) => s.replace(/[\s.,!?~]/g, '').replace(/PC방/g, '피시방').replace(/2명/g, '두명'); function cer(ref: string, hyp: string): number { const a = [...norm(ref)]; const b = [...norm(hyp)]; const d = Array.from({ length: a.length + 1 }, (_, i) => [i, ...new Array(b.length).fill(0)]); for (let j = 1; j <= b.length; j++) d[0][j] = j; for (let i = 1; i <= a.length; i++) for (let j = 1; j <= b.length; j++) d[i][j] = Math.min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1)); return d[a.length][b.length] / Math.max(1, a.length); } const tmp = fs.mkdtempSync(path.join(os.homedir(), '.tmp', 'gvr-eval-')); const store = new Store(path.join(tmp, 'db.sqlite')); const dict = new Dictionary(store, path.join(root, 'app', 'resources'), () => ({})); dict.seed(); const w = new WorkerClient({ cmd: path.join(root, 'worker/.venv/bin/python'), args: ['-m', 'gvr_worker'], cwd: path.join(root, 'worker'), env: { GVR_DATA_DIR: tmp, GVR_MODELS_DIR: modelsDir, GVR_LOG_LEVEL: 'WARNING' } }); w.start(); await new Promise((r) => w.once('ready', r)); const report: string[] = []; for (const model of models) { const t0 = Date.now(); await w.request('load', { slot: 'realtime', model, device, compute_type: computeType }, 30 * 60000); const loadS = (Date.now() - t0) / 1000; for (const useDict of [false, true]) { let sum = 0; let exact = 0; let ms = 0; let unseenSum = 0; let unseenN = 0; const rows: string[] = []; for (const [i, l] of lines.entries()) { const r = await w.request('transcribe', { slot: 'realtime', path: path.join(clipDir, `${i}.ogg`), hotwords: useDict ? dict.hotwords(l.ctx) : '', corrections: useDict ? dict.corrections() : [], dictionary: useDict ? dict.candidates(l.ctx) : [], target_lufs: -18, noise_sensitivity: 0.5, denoise: false, profile_lufs: null, }, 10 * 60000); const hyp = r.text ?? ''; const c = cer(l.text, hyp); sum += c; if (!seenIdx.has(i)) { unseenSum += c; unseenN++; } if (c === 0) exact++; ms += r.elapsed_ms; rows.push(` ${c === 0 ? '=' : c < 0.15 ? '≈' : '≠'} ${l.text} → ${hyp}`); } const line = `${model} ${useDict ? '사전 적용' : '사전 없음'}: CER ${((sum / lines.length) * 100).toFixed(1)}% · 완전 일치 ${exact}/${lines.length} · 평균 ${(ms / lines.length / 1000).toFixed(2)}초/문장 (${device} ${computeType}) · 적재 ${loadS.toFixed(0)}초${seenIdx.size ? ` · 학습에 안 쓴 ${unseenN}문장 CER ${((unseenSum / unseenN) * 100).toFixed(1)}%` : ''}`; console.log(line); console.log(rows.filter((r) => !r.startsWith(' =')).join('\n')); report.push(line); } await w.request('unload', { slot: 'realtime' }); } console.log('\n== 요약 ==\n' + report.join('\n')); await w.stop(); process.exit(0);