- voice-e2e: GVR_E2E_SET=demo(28문장, 문장별 CER·지연), GVR_E2E_MODEL/DEVICE - stt-eval: GVR_EVAL_DEVICE=cuda, 틀린 문장 모두 출력, GVR_EVAL_HELDOUT(학습에 안 쓴 문장 CER) - bot: GVR_RECORD_BOTS 시험 모드에서 봇만 있는 방을 빈 방으로 보고 30초 뒤 나가던 문제 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
81 lines
4.3 KiB
TypeScript
81 lines
4.3 KiB
TypeScript
// M7 인식률 평가: 시연 대화 28문장(TTS)을 모델별로 변환해 CER(글자 오류율)을 잰다. 사전(핫워드+교정) 있음/없음 비교.
|
|
// node --import tsx scripts/stt-eval.mts <models_dir> <model...>
|
|
// GVR_EVAL_HELDOUT=0,3,5 : 파인튜닝 교정에 쓴 문장 번호. 이 문장을 뺀 CER 도 따로 보여 준다
|
|
// GPU: GVR_EVAL_DEVICE=cuda (float16). 리눅스는 LD_LIBRARY_PATH 에 cuBLAS·cuDNN 12/9 경로가 있어야 한다.
|
|
import fs from 'node:fs';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { Store } from '../src/core/db/db.ts';
|
|
import { Dictionary } from '../src/core/dict/dictionary.ts';
|
|
import { WorkerClient } from '../src/core/stt/worker.ts';
|
|
|
|
const [modelsDir, ...models] = process.argv.slice(2);
|
|
const device = process.env.GVR_EVAL_DEVICE === 'cuda' ? 'cuda' : 'cpu';
|
|
const computeType = device === 'cuda' ? 'float16' : 'int8';
|
|
const seenIdx = new Set((process.env.GVR_EVAL_HELDOUT || '').split(',').filter(Boolean).map(Number));
|
|
const root = path.resolve(import.meta.dirname, '..', '..');
|
|
const clipDir = path.join(os.homedir(), '.tmp', 'gvr-demo-clips');
|
|
const demo = fs.readFileSync(path.join(import.meta.dirname, 'demo-stack.mts'), 'utf8');
|
|
const lines = [...demo.matchAll(/\{ u: \d, text: '([^']+)', at: \{ (val|lol|off)/g)].map((m) => ({ text: m[1], ctx: m[2] === 'val' ? 'valorant' : m[2] === 'lol' ? 'lol' : 'none' })) as { text: string; ctx: 'valorant' | 'lol' | 'none' }[];
|
|
const norm = (s: string) => s.replace(/[\s.,!?~]/g, '').replace(/PC방/g, '피시방').replace(/2명/g, '두명');
|
|
function cer(ref: string, hyp: string): number {
|
|
const a = [...norm(ref)];
|
|
const b = [...norm(hyp)];
|
|
const d = Array.from({ length: a.length + 1 }, (_, i) => [i, ...new Array(b.length).fill(0)]);
|
|
for (let j = 1; j <= b.length; j++) d[0][j] = j;
|
|
for (let i = 1; i <= a.length; i++) for (let j = 1; j <= b.length; j++) d[i][j] = Math.min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1));
|
|
return d[a.length][b.length] / Math.max(1, a.length);
|
|
}
|
|
const tmp = fs.mkdtempSync(path.join(os.homedir(), '.tmp', 'gvr-eval-'));
|
|
const store = new Store(path.join(tmp, 'db.sqlite'));
|
|
const dict = new Dictionary(store, path.join(root, 'app', 'resources'), () => ({}));
|
|
dict.seed();
|
|
const w = new WorkerClient({ cmd: path.join(root, 'worker/.venv/bin/python'), args: ['-m', 'gvr_worker'], cwd: path.join(root, 'worker'), env: { GVR_DATA_DIR: tmp, GVR_MODELS_DIR: modelsDir, GVR_LOG_LEVEL: 'WARNING' } });
|
|
w.start();
|
|
await new Promise((r) => w.once('ready', r));
|
|
const report: string[] = [];
|
|
for (const model of models) {
|
|
const t0 = Date.now();
|
|
await w.request('load', { slot: 'realtime', model, device, compute_type: computeType }, 30 * 60000);
|
|
const loadS = (Date.now() - t0) / 1000;
|
|
for (const useDict of [false, true]) {
|
|
let sum = 0;
|
|
let exact = 0;
|
|
let ms = 0;
|
|
let unseenSum = 0;
|
|
let unseenN = 0;
|
|
const rows: string[] = [];
|
|
for (const [i, l] of lines.entries()) {
|
|
const r = await w.request('transcribe', {
|
|
slot: 'realtime',
|
|
path: path.join(clipDir, `${i}.ogg`),
|
|
hotwords: useDict ? dict.hotwords(l.ctx) : '',
|
|
corrections: useDict ? dict.corrections() : [],
|
|
dictionary: useDict ? dict.candidates(l.ctx) : [],
|
|
target_lufs: -18,
|
|
noise_sensitivity: 0.5,
|
|
denoise: false,
|
|
profile_lufs: null,
|
|
}, 10 * 60000);
|
|
const hyp = r.text ?? '';
|
|
const c = cer(l.text, hyp);
|
|
sum += c;
|
|
if (!seenIdx.has(i)) {
|
|
unseenSum += c;
|
|
unseenN++;
|
|
}
|
|
if (c === 0) exact++;
|
|
ms += r.elapsed_ms;
|
|
rows.push(` ${c === 0 ? '=' : c < 0.15 ? '≈' : '≠'} ${l.text} → ${hyp}`);
|
|
}
|
|
const line = `${model} ${useDict ? '사전 적용' : '사전 없음'}: CER ${((sum / lines.length) * 100).toFixed(1)}% · 완전 일치 ${exact}/${lines.length} · 평균 ${(ms / lines.length / 1000).toFixed(2)}초/문장 (${device} ${computeType}) · 적재 ${loadS.toFixed(0)}초${seenIdx.size ? ` · 학습에 안 쓴 ${unseenN}문장 CER ${((unseenSum / unseenN) * 100).toFixed(1)}%` : ''}`;
|
|
console.log(line);
|
|
console.log(rows.filter((r) => !r.startsWith(' =')).join('\n'));
|
|
report.push(line);
|
|
}
|
|
await w.request('unload', { slot: 'realtime' });
|
|
}
|
|
console.log('\n== 요약 ==\n' + report.join('\n'));
|
|
await w.stop();
|
|
process.exit(0);
|