diff --git a/apps/server/package.json b/apps/server/package.json index 108f093..be2db08 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -9,8 +9,13 @@ }, "dependencies": { "@mq/shared": "workspace:*", + "cmu-pronouncing-dictionary": "3.0.0", "hono": "4.13.12", + "kuromoji": "0.1.2", "ulid": "3.0.2", "zod": "4.6.5" + }, + "devDependencies": { + "@types/kuromoji": "^0.1.3" } } diff --git a/apps/server/src/answers/answers.test.ts b/apps/server/src/answers/answers.test.ts new file mode 100644 index 0000000..237b202 --- /dev/null +++ b/apps/server/src/answers/answers.test.ts @@ -0,0 +1,108 @@ +/** 정답 자동 생성 (docs/13): 영어 한글 발음, 일본어 읽는 소리, J-POP 한글 이름. */ +import { expect, test } from 'bun:test'; +import { arpabetToHangul, englishToHangul } from './en-ko'; +import { kanaToHangul, romajiToHangul } from './ja-ko'; +import { readJapanese } from './ja-reading'; +import { wikiLookup } from './wiki'; +import { songAnswersBatch } from '.'; + +test('영어 → 한글: 외래어 표기 규칙과 흔히 쓰는 표기', () => { + const cases: Record = { + Pretender: '프리텐더', + Lemon: '레몬', + 'LOVE DIVE': '러브 다이브', + 'Hype Boy': '하이프 보이', + 'Cheer Up': '치어 업', + 'Blinding Lights': '브라인딩 라이츠', + 'Mr. Simple': '미스터 심플', + 'Shape of You': '셰이프 오브 유', + 'Hotel California': '호텔 캘리포니아', + 'Next Level': '넥스트 레벨', + BTS: '비티에스', + 'I.O.U': '아이 오 유', + Queencard: '퀸카드', + YOASOBI: '요아소비', + 'Kenshi Yonezu': '켄시 요네즈', + 'Good Day': '굿 데이', + }; + for (const [en, ko] of Object.entries(cases)) expect([en, englishToHangul(en)]).toEqual([en, ko]); + expect(arpabetToHangul('K AE1 T')).toBe('캣'); // 짧은 모음 뒤 어말 무성 파열음은 받침 + expect(arpabetToHangul('M EH1 L AH0 N')).toBe('멜런'); // l between vowels doubles +}); + +test('영어 → 한글: a word nobody can read leaves the whole title out; Korean/Japanese text is not English', () => { + expect(englishToHangul('Despacito Xqzt')).toBeNull(); + expect(englishToHangul('좋은 날')).toBeNull(); + expect(englishToHangul('夜に駆ける')).toBeNull(); + expect(englishToHangul('24')).toBeNull(); +}); + +test('가나 → 한글: 표기법(어두 예사소리, つ → 쓰)과 흔한 꼴(거센소리, つ → 츠)', () => { + expect(kanaToHangul('ヨルニ カケル')).toEqual({ rule: '요루니 가케루', common: '요루니 카케루' }); + expect(kanaToHangul('カイブツ')).toEqual({ rule: '가이부쓰', common: '카이부츠' }); + expect(kanaToHangul('コースイ')).toEqual({ rule: '고스이', common: '코스이' }); + expect(kanaToHangul('チョコレート')?.common).toBe('초코레토'); + expect(kanaToHangul('ファイト')?.common).toBe('화이토'); + expect(kanaToHangul('ウッセェワ')?.common).toBe('웃세와'); + expect(kanaToHangul('ABC')).toBeNull(); + expect(romajiToHangul('strawberry')).toBeNull(); +}); + +test('일본어 읽기: kanji through the dictionary process, particles stay on their word, other scripts are null', async () => { + expect(await readJapanese(['夜に駆ける', '雪の華', 'うっせぇわ', 'Official髭男dism', '좋은 날', 'Lemon'])).toEqual(['ヨルニ カケル', 'ユキノ ハナ', 'ウッセェワ', null, null, null]); +}); + +test('위키백과: Korean article names through redirects, disambiguation brackets dropped, 50 names per request', async () => { + const urls: string[] = []; + const fake = (async (url: string) => { + urls.push(url); + return Response.json({ + query: { + normalized: [{ from: 'yoasobi', to: 'Yoasobi' }], + redirects: [{ from: '夜に駆ける', to: '夜に駆ける (曲)' }], + pages: [ + { title: '夜に駆ける (曲)', langlinks: [{ title: '밤을 달리다 (노래)' }] }, + { title: '米津玄師', langlinks: [{ title: '요네즈 켄시' }] }, + { title: 'Yoasobi', langlinks: [{ title: 'YOASOBI' }] }, + { title: '雪の華' }, + ], + }, + }); + }) as unknown as typeof fetch; + const got = await wikiLookup(fake)(['夜に駆ける', '米津玄師', 'yoasobi', '雪の華', ...Array.from({ length: 60 }, (_, i) => `곡${i}`)]); + expect([...got]).toEqual([ + ['夜に駆ける', '밤을 달리다'], + ['米津玄師', '요네즈 켄시'], + ]); + expect(urls).toHaveLength(2); +}); + +test('곡 정답: cleaned title, other names, 한글 발음; J-POP shown by its Korean name with the original and its readings as answers', async () => { + const known: Record = { 夜に駆ける: '밤을 달리다', 米津玄師: '요네즈 켄시' }; + const wiki = async (names: string[]) => new Map(names.flatMap((n): [string, string][] => (known[n] ? [[n, known[n]!]] : []))); + const [pretender, night, flower, plain] = await songAnswersBatch( + null, + [ + { title: 'Pretender (feat. 누구)', artist: 'Official髭男dism' }, + { title: '夜に駆ける', artist: 'YOASOBI' }, + { title: '雪の華', artist: '米津玄師' }, + { title: '좋은 날', artist: '아이유' }, + ], + { korean: true, wiki }, + ); + // no Korean name and no reading for the artist: shown as is and marked 확인 필요 + expect(pretender).toEqual({ title: 'Pretender', artist: 'Official髭男dism', titleAnswers: ['프리텐더'], artistAnswers: [], note: '확인 필요' }); + expect(night).toEqual({ title: '밤을 달리다', artist: 'YOASOBI', titleAnswers: ['夜に駆ける', '요루니 카케루', '요루니 가케루'], artistAnswers: ['요아소비'], note: null }); + // no Korean article: the reading is shown (and said so); the artist's Korean name from Wikipedia + expect(flower).toEqual({ + title: '유키노 하나', + artist: '요네즈 켄시', + titleAnswers: ['雪の華'], + artistAnswers: ['米津玄師', '요네츠 겐시', '요네쓰 겐시'], + note: '한글 이름 자동 변환', + }); + expect(plain).toEqual({ title: '좋은 날', artist: '아이유', titleAnswers: [], artistAnswers: [], note: null }); + // without `korean` (the file fill): the names stay, the answers are the same kind + const [asIs] = await songAnswersBatch(null, [{ title: '夜に駆ける', artist: 'YOASOBI' }]); + expect(asIs).toMatchObject({ title: '夜に駆ける', titleAnswers: ['요루니 카케루', '요루니 가케루'] }); +}); diff --git a/apps/server/src/answers/en-ko.ts b/apps/server/src/answers/en-ko.ts new file mode 100644 index 0000000..8172b3f --- /dev/null +++ b/apps/server/src/answers/en-ko.ts @@ -0,0 +1,195 @@ +/** + * 영어 → 한글 발음 (docs/13): CMU 발음사전(ARPAbet) + 외래어 표기법 영어 규칙을 줄인 것, 흔히 쓰는 표기로 고친 단어표. + * "Pretender" → 프리텐더, "Lemon" → 레몬, "BTS" → 비티에스. 사전에 없는 단어는 일본어 로마자(YOASOBI → 요아소비)로 읽어 보고, + * 그래도 못 읽으면 그 제목은 null (틀린 발음을 정답으로 넣지 않음). + */ +import { romajiToHangul } from './ja-ko'; +import { compose, type Syl } from './hangul'; + +/** The CMU dictionary (~95MB in memory): loaded on the first English title, not at server start. */ +let dict: Record | null = null; +const cmu = (): Record => (dict ??= (require('cmu-pronouncing-dictionary') as { dictionary: Record }).dictionary); + +/** Each vowel as one or two Hangul vowels (AW → 아우, EY → 에이). */ +const VOWEL: Record = { + AA: ['ㅏ'], AE: ['ㅐ'], AH: ['ㅓ'], AO: ['ㅗ'], AW: ['ㅏ', 'ㅜ'], AY: ['ㅏ', 'ㅣ'], EH: ['ㅔ'], ER: ['ㅓ'], EY: ['ㅔ', 'ㅣ'], + IH: ['ㅣ'], IY: ['ㅣ'], OW: ['ㅗ'], OY: ['ㅗ', 'ㅣ'], UH: ['ㅜ'], UW: ['ㅜ'], +}; +const SHORT = new Set(['AE', 'AH', 'EH', 'IH', 'UH', 'AA']); +const ONSET: Record = { + B: 'ㅂ', CH: 'ㅊ', D: 'ㄷ', DH: 'ㄷ', F: 'ㅍ', G: 'ㄱ', HH: 'ㅎ', JH: 'ㅈ', K: 'ㅋ', L: 'ㄹ', M: 'ㅁ', N: 'ㄴ', NG: 'ㅇ', + P: 'ㅍ', R: 'ㄹ', S: 'ㅅ', SH: 'ㅅ', T: 'ㅌ', TH: 'ㅅ', V: 'ㅂ', Z: 'ㅈ', ZH: 'ㅈ', +}; +/** A consonant with no vowel after it gets 으 (or 이/우 below), except these 받침. */ +const CODA: Record = { M: 'ㅁ', N: 'ㄴ', NG: 'ㅇ', L: 'ㄹ' }; +const STOP_CODA: Record = { P: 'ㅂ', T: 'ㅅ', K: 'ㄱ' }; +const Y_GLIDE: Record = { ㅏ: 'ㅑ', ㅓ: 'ㅕ', ㅗ: 'ㅛ', ㅜ: 'ㅠ', ㅔ: 'ㅖ', ㅐ: 'ㅒ', ㅣ: 'ㅣ' }; +const W_GLIDE: Record = { ㅏ: 'ㅘ', ㅓ: 'ㅝ', ㅗ: 'ㅝ', ㅔ: 'ㅞ', ㅐ: 'ㅙ', ㅣ: 'ㅟ', ㅜ: 'ㅜ' }; + +const isVowel = (p: string | undefined) => !!p && p in VOWEL; + +/** ARPAbet ("P R IY0 T EH1 N D ER0") → Hangul ("프리텐더"). */ +export function arpabetToHangul(arpa: string): string { + const raw = arpa.trim().split(/\s+/); + const ph = raw.map((p) => p.replace(/\d$/, '')); + const out: Syl[] = []; + let onset: string | null = null; + let glide: 'Y' | 'W' | null = null; + const lastVowelEnded = () => out.length > 0 && out[out.length - 1]!.jong === ' '; + for (let i = 0; i < ph.length; i++) { + const p = ph[i]!; + const next = ph[i + 1]; + if (isVowel(p)) { + // "simple", "apple", "little": an unstressed vowel before a final l after a stop is not heard (심플, 애플) + if (raw[i] === 'AH0' && next === 'L' && i + 2 === ph.length && onset && ['ㅍ', 'ㅂ', 'ㅋ', 'ㄱ', 'ㅌ', 'ㄷ'].includes(onset) && !glide) { + out.push({ cho: onset, jung: 'ㅡ', jong: 'ㄹ' }); + onset = null; + i++; + continue; + } + // a final schwa reads ㅏ ("drama" → 드라마, "sofa" → 소파) + const vs = raw[i] === 'AH0' && i + 1 === ph.length ? ['ㅏ'] : VOWEL[p]!; + vs.forEach((v, k) => { + let jung = v; + let cho = k === 0 ? (onset ?? 'ㅇ') : 'ㅇ'; + if (k === 0 && glide === 'Y' && !['ㅈ', 'ㅊ'].includes(cho)) jung = Y_GLIDE[jung] ?? jung; + if (k === 0 && glide === 'W') jung = W_GLIDE[jung] ?? jung; + out.push({ cho, jung, jong: ' ' }); + }); + onset = null; + glide = null; + // ER before a vowel: its r is heard ("Erica" → 에리카) + if (p === 'ER' && isVowel(next)) onset = 'ㄹ'; + continue; + } + const vowelNext = isVowel(next) || ((next === 'Y' || next === 'W') && isVowel(ph[i + 2])); + if ((p === 'Y' || p === 'W') && isVowel(next)) { + glide = p; + continue; + } + if (vowelNext) { + if (onset) out.push({ cho: onset, jung: 'ㅡ', jong: ' ' }); + if (p === 'NG') { + if (out.length) out[out.length - 1]!.jong = 'ㅇ'; + onset = 'ㅇ'; + continue; + } + // l between vowels doubles ("melon" → 멜론); r doesn't + if (p === 'L' && i > 0 && lastVowelEnded() && isVowel(ph[i - 1])) out[out.length - 1]!.jong = 'ㄹ'; + onset = ONSET[p] ?? 'ㅇ'; + if (p === 'SH') glide = 'Y'; + continue; + } + // no vowel after this consonant + if (onset) { + out.push({ cho: onset, jung: 'ㅡ', jong: ' ' }); + onset = null; + } + const prev = ph[i - 1]; + if (CODA[p] && lastVowelEnded() && isVowel(prev)) { + out[out.length - 1]!.jong = CODA[p]!; + continue; + } + if (STOP_CODA[p] && lastVowelEnded() && SHORT.has(prev ?? '') && (!next || !['L', 'R', 'M', 'N'].includes(next))) { + out[out.length - 1]!.jong = STOP_CODA[p]!; + continue; + } + // "cheer", "air", "tour": an r after these vowels leaves 어 (치어, 에어, 투어) + if (p === 'R' && ['IH', 'IY', 'EH', 'UH', 'UW'].includes(prev ?? '') && lastVowelEnded()) { + out.push({ cho: 'ㅇ', jung: 'ㅓ', jong: ' ' }); + continue; + } + if (p === 'R' || p === 'HH' || p === 'Y' || p === 'W') continue; + // "lights", "cats": ts → 츠 + if (p === 'T' && next === 'S' && !isVowel(ph[i + 2])) { + out.push({ cho: 'ㅊ', jung: 'ㅡ', jong: ' ' }); + i++; + continue; + } + if (p === 'NG') { + out.push({ cho: 'ㅇ', jung: 'ㅡ', jong: 'ㅇ' }); + continue; + } + const cho = ONSET[p] ?? 'ㅇ'; + const jung = p === 'CH' || p === 'JH' || p === 'ZH' ? 'ㅣ' : p === 'SH' ? (next ? 'ㅠ' : 'ㅣ') : 'ㅡ'; + out.push({ cho, jung, jong: ' ' }); + } + if (onset) out.push({ cho: onset, jung: 'ㅡ', jong: ' ' }); + return compose(out); +} + +/** + * 흔히 쓰는 표기 (국립국어원 외래어 용례·노래 제목에서 흔한 꼴): 규칙으로 읽으면 어색한 단어. + * 규칙: lemon → 레먼, of → 어브, on → 안 … + */ +const COMMON: Record = { + lemon: '레몬', of: '오브', on: '온', off: '오프', oh: '오', ok: '오케이', okay: '오케이', yeah: '예', baby: '베이비', love: '러브', + sorry: '쏘리', boy: '보이', girl: '걸', girls: '걸스', boys: '보이즈', you: '유', your: '유어', "you're": '유어', my: '마이', me: '미', + the: '더', a: '어', an: '언', to: '투', for: '포', from: '프롬', with: '위드', all: '올', so: '쏘', no: '노', not: '낫', + hot: '핫', stop: '스톱', top: '탑', pop: '팝', shop: '샵', rock: '록', body: '바디', party: '파티', story: '스토리', + music: '뮤직', radio: '라디오', video: '비디오', super: '슈퍼', dynamite: '다이너마이트', butter: '버터', piano: '피아노', + coffee: '커피', chocolate: '초콜릿', chocolat: '쇼콜라', alone: '얼론', again: '어게인', about: '어바웃', ah: '아', + go: '고', home: '홈', hello: '헬로', bye: '바이', goodbye: '굿바이', merry: '메리', christmas: '크리스마스', valentine: '발렌타인', + wanna: '워너', gonna: '고나', gotta: '가라', what: '왓', want: '원트', water: '워터', world: '월드', word: '워드', work: '워크', + only: '온리', one: '원', once: '원스', two: '투', three: '쓰리', four: '포', five: '파이브', six: '식스', seven: '세븐', + eight: '에잇', nine: '나인', ten: '텐', zero: '제로', hundred: '헌드레드', mr: '미스터', mrs: '미세스', ms: '미즈', dr: '닥터', + vs: '버서스', feat: '피처링', tv: '티비', dj: '디제이', mc: '엠씨', ost: '오에스티', ufo: '유에프오', + good: '굿', god: '갓', level: '레벨', angel: '엔젤', idol: '아이돌', hotel: '호텔', eleven: '일레븐', new: '뉴', news: '뉴스', rollin: '롤린', + rhapsody: '랩소디', bohemian: '보헤미안', scenario: '시나리오', spirit: '스피릿', someone: '썸원', something: '썸씽', + celebrity: '셀러브리티', without: '위드아웃', california: '캘리포니아', supernova: '슈퍼노바', drama: '드라마', banana: '바나나', + energy: '에너지', fantasy: '판타지', memory: '메모리', melody: '멜로디', lady: '레이디', tonight: '투나잇', night: '나이트', + light: '라이트', right: '라이트', white: '화이트', heart: '하트', start: '스타트', smart: '스마트', bang: '뱅', gang: '갱', + cool: '쿨', school: '스쿨', fire: '파이어', wonder: '원더', winter: '윈터', summer: '썸머', sun: '썬', sunny: '써니', + catch: '캐치', match: '매치', fashion: '패션', mission: '미션', vision: '비전', question: '퀘스천', tell: '텔', bell: '벨', +}; + +const LETTER: Record = { + a: '에이', b: '비', c: '씨', d: '디', e: '이', f: '에프', g: '지', h: '에이치', i: '아이', j: '제이', k: '케이', l: '엘', m: '엠', + n: '엔', o: '오', p: '피', q: '큐', r: '알', s: '에스', t: '티', u: '유', v: '브이', w: '더블유', x: '엑스', y: '와이', z: '지', +}; + +/** One English word → Hangul, or null when it can't be read. */ +export function wordToHangul(raw: string): string | null { + const dictionary = cmu(); + const w = raw.normalize('NFKC').replace(/[’‘`]/g, "'").replace(/^[^A-Za-z0-9]+|[^A-Za-z0-9']+$/g, ''); + if (!w) return ''; + if (/^\d+$/.test(w)) return w; + const low = w.toLowerCase().replace(/'$/, ''); + if (COMMON[low]) return COMMON[low]!; + // initials: "BTS", "R.O.C.K", "I.O.U" + if (/^(?:[A-Z]\.?){1,6}$/.test(w) && (w.length <= 4 || w.includes('.')) && !dictionary[low]) return [...w.replace(/\./g, '').toLowerCase()].map((c) => LETTER[c]).join(''); + const arpa = dictionary[low]; + if (arpa) return arpabetToHangul(arpa); + // "can't"-style words missing from the dictionary: read the part before the apostrophe + if (low.includes("'")) { + const head = dictionary[low.split("'")[0]!]; + if (head) return arpabetToHangul(head); + } + const jp = romajiToHangul(low); + if (jp) return jp.common; + // two words written as one ("Queencard", "Antifragile") + for (let k = 2; k <= low.length - 2; k++) { + const a = COMMON[low.slice(0, k)] ?? (dictionary[low.slice(0, k)] ? arpabetToHangul(dictionary[low.slice(0, k)]!) : null); + const b = a && (COMMON[low.slice(k)] ?? (dictionary[low.slice(k)] ? arpabetToHangul(dictionary[low.slice(k)]!) : null)); + if (a && b) return a + b; + } + return null; +} + +/** A title in Latin letters → Hangul words joined by spaces; null when a word can't be read or there are no Latin letters. */ +export function englishToHangul(text: string): string | null { + const t = text.normalize('NFKC'); + if (!/[A-Za-z]/.test(t) || /[\p{Script=Hangul}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Han}]/u.test(t)) return null; + const words = t + .replace(/&/g, ' and ') + .split(/[\s\-_/,.!?~:;()"[\]{}]+(?=\S)|\s+/) + .flatMap((w) => (/^(?:[A-Z]\.){2,}[A-Z]?\.?$/.test(w) ? [w] : w.split(/(?<=[a-z])\.(?=[a-z])/i))) + .filter((w) => /[A-Za-z0-9]/.test(w)); + const out: string[] = []; + for (const w of words) { + const h = wordToHangul(w); + if (h === null) return null; + if (h) out.push(h); + } + return out.length ? out.join(' ') : null; +} diff --git a/apps/server/src/answers/hangul.ts b/apps/server/src/answers/hangul.ts new file mode 100644 index 0000000..d764070 --- /dev/null +++ b/apps/server/src/answers/hangul.ts @@ -0,0 +1,15 @@ +/** Hangul syllables from jamo (초성·중성·종성), shared by the English and Japanese readers. */ +const CHO = 'ㄱㄲㄴㄷㄸㄹㅁㅂㅃㅅㅆㅇㅈㅉㅊㅋㅌㅍㅎ'; +const JUNG = 'ㅏㅐㅑㅒㅓㅔㅕㅖㅗㅘㅙㅚㅛㅜㅝㅞㅟㅠㅡㅢㅣ'; +const JONG = ' ㄱㄲㄳㄴㄵㄶㄷㄹㄺㄻㄼㄽㄾㄿㅀㅁㅂㅄㅅㅆㅇㅈㅊㅋㅌㅍㅎ'; + +export interface Syl { + cho: string; + jung: string; + jong: string; +} + +export function compose(list: Syl[]): string { + return list.map((s) => String.fromCharCode(0xac00 + (CHO.indexOf(s.cho) * 21 + JUNG.indexOf(s.jung)) * 28 + Math.max(0, JONG.indexOf(s.jong)))).join(''); +} + diff --git a/apps/server/src/answers/index.ts b/apps/server/src/answers/index.ts index f71f827..bd598f6 100644 --- a/apps/server/src/answers/index.ts +++ b/apps/server/src/answers/index.ts @@ -2,25 +2,43 @@ * 정답 자동 생성 (docs/13): for a song's title and artist, the answers people may type besides the names shown. * - the title without brackets / feat. / version marks (cleanTitle) and the other names that were in brackets * - the aliases the server DB already has for that song and its artists - * The names shown (제목·가수) always count on their own, so they are never listed again. + * - 영어 → 한글 발음 (Pretender → 프리텐더), 일본어 → 원어 + 읽는 소리 두 가지 (夜に駆ける → 요루니 가케루 / 요루니 카케루) + * With `korean`, a Japanese title/artist is shown by its Korean name: 위키백과 한국어 문서 이름 → 괄호 안 한글 제목 → + * 읽는 소리 (note '한글 이름 자동 변환'); nothing found → note '확인 필요'. The names shown always count on their own. */ import type { Database } from 'bun:sqlite'; import { cleanTitle } from '@mq/shared'; +import { englishToHangul } from './en-ko'; +import { kanaToHangul } from './ja-ko'; +import { hasJapanese, readJapanese, type JaReader } from './ja-reading'; +import type { WikiLookup } from './wiki'; export interface SongAnswers { - /** The title to show (cleaned). */ + /** The title to show (cleaned; the Korean name of a Japanese title with `korean`). */ title: string; + /** The artist to show (the Korean name of a Japanese artist with `korean`). */ + artist: string; titleAnswers: string[]; artistAnswers: string[]; + /** '한글 이름 자동 변환' / '확인 필요' for a Japanese name shown in Korean, else null. */ + note: string | null; +} + +export interface AnswerOptions { + /** Show Japanese titles/artists by their Korean names (playlist rows, DB cleanup). */ + korean?: boolean; + wiki?: WikiLookup; + read?: JaReader; } const MAX = 30; +const HANGUL = /\p{Script=Hangul}/u; -function uniq(list: string[], except: string): string[] { - const seen = new Set([except.trim().toLowerCase()]); +function uniq(list: (string | null | undefined)[], except: string[]): string[] { + const seen = new Set(except.map((x) => x.trim().toLowerCase())); const out: string[] = []; for (const raw of list) { - const a = raw.replace(/\s+/g, ' ').trim().slice(0, 200); + const a = (raw ?? '').replace(/\s+/g, ' ').trim().slice(0, 200); const k = a.toLowerCase(); if (!a || seen.has(k)) continue; seen.add(k); @@ -41,12 +59,69 @@ export function dbAliases(db: Database, songId: number): { title: string[]; arti }; } -export function songAnswers(db: Database | null, s: { title: string; artist: string; songId?: number | null }): SongAnswers { - const clean = cleanTitle(s.title); - const fromDb = db && s.songId ? dbAliases(db, s.songId) : { title: [], artist: [] }; - return { - title: clean.title, - titleAnswers: uniq([...clean.aliases, ...fromDb.title], clean.title), - artistAnswers: uniq(fromDb.artist, s.artist), +/** The names in an artist credit ("A, B & C"); the whole credit too. */ +const artistNames = (a: string) => [...new Set([a, ...a.split(/\s*(?:,|&|\/|·|\sx\s|\bfeat\.?\s|\bwith\s)\s*/i)].map((x) => x.trim()).filter(Boolean))]; + +/** Answers for many songs at once (one Japanese reading process, one Wikipedia request per 50 names). */ +export async function songAnswersBatch(db: Database | null, songs: { title: string; artist: string; songId?: number | null }[], opts: AnswerOptions = {}): Promise { + const read = opts.read ?? readJapanese; + const clean = songs.map((s) => cleanTitle(s.title)); + // every Japanese text whose reading we want: titles, their bracket names, artist names + const jaTexts = new Set(); + songs.forEach((s, i) => { + for (const t of [clean[i]!.title, ...clean[i]!.aliases]) if (hasJapanese(t)) jaTexts.add(t); + for (const n of artistNames(s.artist)) if (hasJapanese(n)) jaTexts.add(n); + }); + const jaList = [...jaTexts]; + const readings = new Map(); + if (jaList.length) { + const got = await read(jaList); + jaList.forEach((t, i) => { + const k = got[i] ? kanaToHangul(got[i]!) : null; + if (k) readings.set(t, k); + }); + } + const wikiNames = opts.korean && opts.wiki && jaList.length ? await opts.wiki(jaList).catch(() => new Map()) : new Map(); + const say = (t: string): string[] => { + if (hasJapanese(t)) { + const r = readings.get(t); + return r ? [r.common, r.rule] : []; + } + if (HANGUL.test(t)) return []; + const e = englishToHangul(t); + return e ? [e] : []; }; + return songs.map((s, i) => { + const c = clean[i]!; + const fromDb = db && s.songId ? dbAliases(db, s.songId) : { title: [], artist: [] }; + let title = c.title; + let artist = s.artist; + const notes: string[] = []; + const extraTitle: string[] = []; + const extraArtist: string[] = []; + // a Japanese name shown in Korean: 위키백과 → 괄호 안 한글 이름 → 읽는 소리 + const korean = (name: string, bracket: string | undefined, push: string[]): string => { + const ko = wikiNames.get(name) ?? bracket ?? readings.get(name)?.common; + if (!ko) { + notes.push('확인 필요'); + return name; + } + if (!wikiNames.has(name) && !bracket) notes.push('한글 이름 자동 변환'); + push.push(name); + return ko; + }; + if (opts.korean && hasJapanese(title)) title = korean(title, c.aliases.find((a) => HANGUL.test(a) && !hasJapanese(a)), extraTitle); + if (opts.korean && hasJapanese(artist)) artist = korean(artist, undefined, extraArtist); + const titleAnswers = uniq([...c.aliases, ...extraTitle, ...fromDb.title, ...[c.title, ...c.aliases, ...extraTitle, ...fromDb.title].flatMap(say)], [title]); + const names = artistNames(s.artist); + const artistAnswers = uniq([...extraArtist, ...fromDb.artist, ...names.flatMap(say), ...fromDb.artist.flatMap(say)], [artist, s.artist === artist ? '' : s.artist].filter(Boolean)); + // the artist as written stays an answer when it was replaced by its Korean name + if (artist !== s.artist && !artistAnswers.includes(s.artist)) artistAnswers.unshift(s.artist); + return { title, artist, titleAnswers, artistAnswers: artistAnswers.slice(0, MAX), note: notes.length ? [...new Set(notes)].join(' · ') : null }; + }); +} + +/** One song (no Korean renaming, no network): the file fill and tests. */ +export async function songAnswers(db: Database | null, s: { title: string; artist: string; songId?: number | null }, opts: AnswerOptions = {}): Promise { + return (await songAnswersBatch(db, [s], opts))[0]!; } diff --git a/apps/server/src/answers/ja-ko.ts b/apps/server/src/answers/ja-ko.ts new file mode 100644 index 0000000..3c67c53 --- /dev/null +++ b/apps/server/src/answers/ja-ko.ts @@ -0,0 +1,128 @@ +/** + * 일본어 → 한글 소리 (docs/13): 가나를 외래어 표기법 일본어 표로 한글로 바꿈. + * 두 가지를 같이 냄: rule = 표기법(단어 첫소리는 예사소리, つ → 쓰: 요루니 가케루), common = 흔히 쓰는 꼴(거센소리, つ → 츠: 요루니 카케루). + * 한자 읽기는 ja-reading.ts(형태소 분석 사전)가 가나로 바꿔 줌. 로마자 이름(YOASOBI, Yonezu Kenshi)도 같은 표로 읽음. + */ +import { compose } from './hangul'; + +export interface KoReading { + rule: string; + common: string; +} + +/** Katakana row → [onset as 표기법 word-initial, onset elsewhere / common]. */ +const ROW: Record = { + // kana: [plain onset, aspirated onset, vowel] + ア: ['ㅇ', 'ㅇ', 'ㅏ'], イ: ['ㅇ', 'ㅇ', 'ㅣ'], ウ: ['ㅇ', 'ㅇ', 'ㅜ'], エ: ['ㅇ', 'ㅇ', 'ㅔ'], オ: ['ㅇ', 'ㅇ', 'ㅗ'], + カ: ['ㄱ', 'ㅋ', 'ㅏ'], キ: ['ㄱ', 'ㅋ', 'ㅣ'], ク: ['ㄱ', 'ㅋ', 'ㅜ'], ケ: ['ㄱ', 'ㅋ', 'ㅔ'], コ: ['ㄱ', 'ㅋ', 'ㅗ'], + ガ: ['ㄱ', 'ㄱ', 'ㅏ'], ギ: ['ㄱ', 'ㄱ', 'ㅣ'], グ: ['ㄱ', 'ㄱ', 'ㅜ'], ゲ: ['ㄱ', 'ㄱ', 'ㅔ'], ゴ: ['ㄱ', 'ㄱ', 'ㅗ'], + サ: ['ㅅ', 'ㅅ', 'ㅏ'], シ: ['ㅅ', 'ㅅ', 'ㅣ'], ス: ['ㅅ', 'ㅅ', 'ㅡ'], セ: ['ㅅ', 'ㅅ', 'ㅔ'], ソ: ['ㅅ', 'ㅅ', 'ㅗ'], + ザ: ['ㅈ', 'ㅈ', 'ㅏ'], ジ: ['ㅈ', 'ㅈ', 'ㅣ'], ズ: ['ㅈ', 'ㅈ', 'ㅡ'], ゼ: ['ㅈ', 'ㅈ', 'ㅔ'], ゾ: ['ㅈ', 'ㅈ', 'ㅗ'], + タ: ['ㄷ', 'ㅌ', 'ㅏ'], チ: ['ㅈ', 'ㅊ', 'ㅣ'], ツ: ['ㅆ', 'ㅊ', 'ㅡ'], テ: ['ㄷ', 'ㅌ', 'ㅔ'], ト: ['ㄷ', 'ㅌ', 'ㅗ'], + ダ: ['ㄷ', 'ㄷ', 'ㅏ'], ヂ: ['ㅈ', 'ㅈ', 'ㅣ'], ヅ: ['ㅈ', 'ㅈ', 'ㅡ'], デ: ['ㄷ', 'ㄷ', 'ㅔ'], ド: ['ㄷ', 'ㄷ', 'ㅗ'], + ナ: ['ㄴ', 'ㄴ', 'ㅏ'], ニ: ['ㄴ', 'ㄴ', 'ㅣ'], ヌ: ['ㄴ', 'ㄴ', 'ㅜ'], ネ: ['ㄴ', 'ㄴ', 'ㅔ'], ノ: ['ㄴ', 'ㄴ', 'ㅗ'], + ハ: ['ㅎ', 'ㅎ', 'ㅏ'], ヒ: ['ㅎ', 'ㅎ', 'ㅣ'], フ: ['ㅎ', 'ㅎ', 'ㅜ'], ヘ: ['ㅎ', 'ㅎ', 'ㅔ'], ホ: ['ㅎ', 'ㅎ', 'ㅗ'], + バ: ['ㅂ', 'ㅂ', 'ㅏ'], ビ: ['ㅂ', 'ㅂ', 'ㅣ'], ブ: ['ㅂ', 'ㅂ', 'ㅜ'], ベ: ['ㅂ', 'ㅂ', 'ㅔ'], ボ: ['ㅂ', 'ㅂ', 'ㅗ'], + パ: ['ㅍ', 'ㅍ', 'ㅏ'], ピ: ['ㅍ', 'ㅍ', 'ㅣ'], プ: ['ㅍ', 'ㅍ', 'ㅜ'], ペ: ['ㅍ', 'ㅍ', 'ㅔ'], ポ: ['ㅍ', 'ㅍ', 'ㅗ'], + マ: ['ㅁ', 'ㅁ', 'ㅏ'], ミ: ['ㅁ', 'ㅁ', 'ㅣ'], ム: ['ㅁ', 'ㅁ', 'ㅜ'], メ: ['ㅁ', 'ㅁ', 'ㅔ'], モ: ['ㅁ', 'ㅁ', 'ㅗ'], + ヤ: ['ㅇ', 'ㅇ', 'ㅑ'], ユ: ['ㅇ', 'ㅇ', 'ㅠ'], ヨ: ['ㅇ', 'ㅇ', 'ㅛ'], + ラ: ['ㄹ', 'ㄹ', 'ㅏ'], リ: ['ㄹ', 'ㄹ', 'ㅣ'], ル: ['ㄹ', 'ㄹ', 'ㅜ'], レ: ['ㄹ', 'ㄹ', 'ㅔ'], ロ: ['ㄹ', 'ㄹ', 'ㅗ'], + ワ: ['ㅇ', 'ㅇ', 'ㅘ'], ヰ: ['ㅇ', 'ㅇ', 'ㅣ'], ヱ: ['ㅇ', 'ㅇ', 'ㅔ'], ヲ: ['ㅇ', 'ㅇ', 'ㅗ'], ヴ: ['ㅂ', 'ㅂ', 'ㅜ'], +}; +const SMALL_Y: Record = { ャ: 'ㅑ', ュ: 'ㅠ', ョ: 'ㅛ' }; +const SMALL_V: Record = { ァ: 'ㅏ', ィ: 'ㅣ', ゥ: 'ㅜ', ェ: 'ㅔ', ォ: 'ㅗ' }; +/** ファ → 화, ウィ → 위: the small vowel after フ/ウ/ヴ makes a w/ㅎ glide. */ +const W_OF: Record = { ㅏ: 'ㅘ', ㅣ: 'ㅟ', ㅔ: 'ㅞ', ㅗ: 'ㅝ', ㅜ: 'ㅜ' }; +const PLAIN_Y: Record = { ㅑ: 'ㅏ', ㅠ: 'ㅜ', ㅛ: 'ㅗ' }; + +export const toKatakana = (s: string) => s.normalize('NFKC').replace(/[ぁ-ゖ]/g, (c) => String.fromCharCode(c.charCodeAt(0) + 0x60)); +export const isKana = (s: string) => /^[ァ-ーー・\s]+$/.test(toKatakana(s)); + +/** Katakana words (spaces between words) → both readings, or null when something is not kana. */ +export function kanaToHangul(words: string): KoReading | null { + const out: { rule: string[]; common: string[] } = { rule: [], common: [] }; + for (const word of toKatakana(words).split(/[\s・]+/).filter(Boolean)) { + const syl: { cho: [string, string]; jung: string; jong: string }[] = []; + for (const ch of word) { + const last = syl[syl.length - 1]; + if (ch === 'ー') continue; + if (ch === 'ン') { + if (!last) return null; + last.jong = 'ㄴ'; + continue; + } + if (ch === 'ッ') { + if (last) last.jong = 'ㅅ'; + continue; + } + if (SMALL_Y[ch]) { + if (!last || last.jung !== 'ㅣ') return null; + // ジャ → 자, チャ → 차 (no ㅈ+ㅑ) + last.jung = ['ㅈ', 'ㅊ'].includes(last.cho[1]) ? PLAIN_Y[SMALL_Y[ch]!]! : SMALL_Y[ch]!; + continue; + } + if (SMALL_V[ch]) { + if (!last) return null; + const v = SMALL_V[ch]!; + // ファ → 화, ウィ → 위; ティ → 티, ヴァ → 바 + last.jung = last.cho[1] === 'ㅎ' || last.cho[1] === 'ㅇ' ? (W_OF[v] ?? v) : v; + continue; + } + const r = ROW[ch]; + if (!r) return null; + syl.push({ cho: [r[0], r[1]], jung: r[2], jong: ' ' }); + } + if (!syl.length) continue; + out.rule.push(compose(syl.map((s, i) => ({ cho: i === 0 ? s.cho[0] : s.cho[1] === 'ㅊ' && s.cho[0] === 'ㅆ' ? 'ㅆ' : s.cho[1], jung: s.jung, jong: s.jong })))); + out.common.push(compose(syl.map((s) => ({ cho: s.cho[1], jung: s.jung, jong: s.jong })))); + } + if (!out.rule.length) return null; + return { rule: out.rule.join(' '), common: out.common.join(' ') }; +} + +/** Hepburn (and common wāpuro) romaji → katakana, or null when the word isn't romaji. */ +const ROMA: [string, string][] = [ + ['kya', 'キャ'], ['kyu', 'キュ'], ['kyo', 'キョ'], ['gya', 'ギャ'], ['gyu', 'ギュ'], ['gyo', 'ギョ'], ['sha', 'シャ'], ['shu', 'シュ'], ['sho', 'ショ'], ['shi', 'シ'], + ['she', 'シェ'], ['ja', 'ジャ'], ['ju', 'ジュ'], ['jo', 'ジョ'], ['ji', 'ジ'], ['je', 'ジェ'], ['cha', 'チャ'], ['chu', 'チュ'], ['cho', 'チョ'], ['chi', 'チ'], ['che', 'チェ'], + ['tsu', 'ツ'], ['nya', 'ニャ'], ['nyu', 'ニュ'], ['nyo', 'ニョ'], ['hya', 'ヒャ'], ['hyu', 'ヒュ'], ['hyo', 'ヒョ'], ['mya', 'ミャ'], ['myu', 'ミュ'], ['myo', 'ミョ'], + ['rya', 'リャ'], ['ryu', 'リュ'], ['ryo', 'リョ'], ['bya', 'ビャ'], ['byu', 'ビュ'], ['byo', 'ビョ'], ['pya', 'ピャ'], ['pyu', 'ピュ'], ['pyo', 'ピョ'], ['fu', 'フ'], + ['ka', 'カ'], ['ki', 'キ'], ['ku', 'ク'], ['ke', 'ケ'], ['ko', 'コ'], ['ga', 'ガ'], ['gi', 'ギ'], ['gu', 'グ'], ['ge', 'ゲ'], ['go', 'ゴ'], + ['sa', 'サ'], ['si', 'シ'], ['su', 'ス'], ['se', 'セ'], ['so', 'ソ'], ['za', 'ザ'], ['zi', 'ジ'], ['zu', 'ズ'], ['ze', 'ゼ'], ['zo', 'ゾ'], + ['ta', 'タ'], ['ti', 'チ'], ['tu', 'ツ'], ['te', 'テ'], ['to', 'ト'], ['da', 'ダ'], ['di', 'ヂ'], ['du', 'ヅ'], ['de', 'デ'], ['do', 'ド'], + ['na', 'ナ'], ['ni', 'ニ'], ['nu', 'ヌ'], ['ne', 'ネ'], ['no', 'ノ'], ['ha', 'ハ'], ['hi', 'ヒ'], ['hu', 'フ'], ['he', 'ヘ'], ['ho', 'ホ'], + ['ba', 'バ'], ['bi', 'ビ'], ['bu', 'ブ'], ['be', 'ベ'], ['bo', 'ボ'], ['pa', 'パ'], ['pi', 'ピ'], ['pu', 'プ'], ['pe', 'ペ'], ['po', 'ポ'], + ['ma', 'マ'], ['mi', 'ミ'], ['mu', 'ム'], ['me', 'メ'], ['mo', 'モ'], ['ya', 'ヤ'], ['yu', 'ユ'], ['yo', 'ヨ'], + ['ra', 'ラ'], ['ri', 'リ'], ['ru', 'ル'], ['re', 'レ'], ['ro', 'ロ'], ['wa', 'ワ'], ['wo', 'ヲ'], + ['a', 'ア'], ['i', 'イ'], ['u', 'ウ'], ['e', 'エ'], ['o', 'オ'], +]; + +export function romajiToKana(word: string): string | null { + let w = word.toLowerCase().replace(/[āâ]/g, 'aa').replace(/[ūû]/g, 'uu').replace(/[ōô]/g, 'ou').replace(/[^a-z]/g, ''); + if (w.length < 2) return null; + let out = ''; + while (w) { + // doubled consonant → ッ ("kk", "tch") + if (/^([kgsztdhbpcfj])\1/.test(w) || /^tch/.test(w)) { + out += 'ッ'; + w = w.slice(1); + continue; + } + // n before a consonant or at the end → ン ("n'" too) + if (/^n(?![aiueoy])/.test(w)) { + out += 'ン'; + w = w.slice(1); + continue; + } + const hit = ROMA.find(([r]) => w.startsWith(r)); + if (!hit) return null; + out += hit[1]; + w = w.slice(hit[0].length); + } + return out; +} + +/** A romaji word ("yoasobi", "kenshi") → both readings, or null when it doesn't read as romaji. */ +export function romajiToHangul(word: string): KoReading | null { + const k = romajiToKana(word); + return k ? kanaToHangul(k) : null; +} diff --git a/apps/server/src/answers/ja-reading.ts b/apps/server/src/answers/ja-reading.ts new file mode 100644 index 0000000..a5da0e6 --- /dev/null +++ b/apps/server/src/answers/ja-reading.ts @@ -0,0 +1,27 @@ +/** + * 일본어 제목·가수의 읽는 소리 (docs/13): ja-worker.ts in a separate bun process (its dictionary is too big to keep in the + * server), one process per batch. Texts without kana or kanji are not sent. Fails soft: null for every text. + */ +import { join } from 'node:path'; + +export const hasJapanese = (s: string) => /[\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Han}]/u.test(s) && !/\p{Script=Hangul}/u.test(s); + +export type JaReader = (texts: string[]) => Promise<(string | null)[]>; + +export const readJapanese: JaReader = async (texts) => { + const want = texts.map((t, i) => [t, i] as const).filter(([t]) => hasJapanese(t)); + const out: (string | null)[] = texts.map(() => null); + if (!want.length) return out; + try { + const p = Bun.spawn([process.execPath, join(import.meta.dir, 'ja-worker.ts')], { stdin: new TextEncoder().encode(JSON.stringify(want.map(([t]) => t))), stdout: 'pipe', stderr: 'pipe' }); + const timer = setTimeout(() => p.kill(), 60_000); + const [text, code] = await Promise.all([new Response(p.stdout).text(), p.exited]); + clearTimeout(timer); + if (code !== 0) return out; + const got = JSON.parse(text) as (string | null)[]; + want.forEach(([, i], k) => (out[i] = got[k] ?? null)); + } catch { + /* no readings: the other answers still count */ + } + return out; +}; diff --git a/apps/server/src/answers/ja-worker.ts b/apps/server/src/answers/ja-worker.ts new file mode 100644 index 0000000..e6dca36 --- /dev/null +++ b/apps/server/src/answers/ja-worker.ts @@ -0,0 +1,26 @@ +/** + * 일본어 읽기 (docs/13), run as its own short process by ja-reading.ts: the morphological dictionary (kuromoji) takes + * ~350MB, so the server never loads it itself. Reads JSON string[] on stdin, writes (string | null)[] on stdout: each text's + * pronunciation in katakana, words separated by spaces (particles stay on their word), null when it has other letters. + */ +import kuromoji from 'kuromoji'; +import { dirname, join } from 'node:path'; + +const texts = JSON.parse(await Bun.stdin.text()) as string[]; +const dicPath = join(dirname(require.resolve('kuromoji')), '..', 'dict'); +const tokenizer = await new Promise>((ok, fail) => kuromoji.builder({ dicPath }).build((e, t) => (e ? fail(e) : ok(t)))); +const KANA = /^[ァ-ーー]+$/; +const toKata = (s: string) => s.replace(/[ぁ-ゖ]/g, (c) => String.fromCharCode(c.charCodeAt(0) + 0x60)); +const out = texts.map((text) => { + const words: string[] = []; + for (const t of tokenizer.tokenize(text.normalize('NFKC'))) { + if (t.pos === '記号' || /^[\s\p{P}\p{S}]+$/u.test(t.surface_form)) continue; + const say = toKata(t.pronunciation && t.pronunciation !== '*' ? t.pronunciation : t.reading && t.reading !== '*' ? t.reading : t.surface_form); + if (!KANA.test(say)) return null; + const attach = words.length > 0 && (/^[ァィゥェォャュョッー]/.test(say) || t.pos === '助詞' || t.pos === '助動詞' || t.pos_detail_1 === '接尾' || t.pos_detail_1 === '非自立'); + if (attach) words[words.length - 1] += say; + else words.push(say); + } + return words.length ? words.join(' ') : null; +}); +process.stdout.write(JSON.stringify(out)); diff --git a/apps/server/src/answers/wiki.ts b/apps/server/src/answers/wiki.ts new file mode 100644 index 0000000..55314e3 --- /dev/null +++ b/apps/server/src/answers/wiki.ts @@ -0,0 +1,32 @@ +/** + * J-POP 한글 이름 (docs/13): 일본어 위키백과 문서의 한국어 문서 이름 (米津玄師 → 요네즈 켄시, 夜に駆ける → 밤을 달리다). + * 50 names per request, redirects followed, "(노래)"·"(가수)" 같은 구분 괄호는 뗌. Names without a Korean article are left out. + */ +export type WikiLookup = (names: string[]) => Promise>; + +const API = 'https://ja.wikipedia.org/w/api.php'; + +export function wikiLookup(fetchImpl: typeof fetch = fetch): WikiLookup { + return async (names) => { + const out = new Map(); + const list = [...new Set(names.map((n) => n.trim()).filter((n) => n && !/[|#<>[\]{}]/.test(n)))]; + for (let i = 0; i < list.length; i += 50) { + const chunk = list.slice(i, i + 50); + const url = `${API}?action=query&format=json&formatversion=2&redirects=1&prop=langlinks&lllang=ko&lllimit=max&titles=${encodeURIComponent(chunk.join('|'))}`; + const res = await fetchImpl(url, { headers: { 'user-agent': 'music-quiz/1.0 (https://mquiz.tkrmagid.kr)' }, signal: AbortSignal.timeout(15_000) }); + if (!res.ok) throw new Error(`위키백과 ${res.status}`); + const j = (await res.json()) as { + query?: { normalized?: { from: string; to: string }[]; redirects?: { from: string; to: string }[]; pages?: { title: string; langlinks?: { title: string }[] }[] }; + }; + const q = j.query ?? {}; + const ko = new Map((q.pages ?? []).flatMap((p) => (p.langlinks?.[0]?.title ? [[p.title, p.langlinks[0].title] as const] : []))); + const step = (m: { from: string; to: string }[] | undefined, t: string) => m?.find((x) => x.from === t)?.to ?? t; + for (const name of chunk) { + const page = step(q.redirects, step(q.normalized, name)); + const title = ko.get(page)?.replace(/\s*\([^)]*\)\s*$/, '').trim(); + if (title && /\p{Script=Hangul}/u.test(title)) out.set(name, title); + } + } + return out; + }; +} diff --git a/apps/server/src/custom-quiz.test.ts b/apps/server/src/custom-quiz.test.ts index 64ffc7b..f4ffc9d 100644 --- a/apps/server/src/custom-quiz.test.ts +++ b/apps/server/src/custom-quiz.test.ts @@ -215,7 +215,7 @@ test('플레이리스트 가져오기: DB songs by video, cleaned YouTube titles expect(rows[2]).toMatchObject({ status: 'found', videoId: 'hypeboy0001' }); expect(rows[3]).toMatchObject({ status: 'missing', videoId: null }); // answers besides the shown names (docs/13): the DB song's aliases come along - expect(rows[0]).toMatchObject({ titleAnswers: ['Good Day'], artistAnswers: ['IU'] }); + expect(rows[0]).toMatchObject({ titleAnswers: ['Good Day', '굿 데이'], artistAnswers: ['IU'] }); // someone else's job is not visible const g = await guest(s.base, '손님'); expect((await api(s.base, `/api/admin/quizzes/pick/playlist/${job.id}`, { cookie: g.cookie })).res.status).toBe(403); @@ -230,8 +230,8 @@ test('정답 자동 채우기 (docs/13): cleaned-title names and the aliases of body: JSON.stringify({ songs: [{ title: '좋은 날', artist: '아이유' }, { title: '사랑을 했다(LOVE SCENARIO)', artist: '아이콘' }, { title: '모르는 곡' }] }), }); expect(r.body.answers).toEqual([ - { titleAnswers: ['Good Day'], artistAnswers: ['IU'] }, - { titleAnswers: ['LOVE SCENARIO', '사랑을 했다 LOVE SCENARIO'], artistAnswers: [] }, + { titleAnswers: ['Good Day', '굿 데이'], artistAnswers: ['IU'] }, + { titleAnswers: ['LOVE SCENARIO', '사랑을 했다 LOVE SCENARIO', '러브 시나리오'], artistAnswers: [] }, { titleAnswers: [], artistAnswers: [] }, ]); const g = await guest(s.base, '손님'); diff --git a/apps/server/src/http/admin-quizzes.ts b/apps/server/src/http/admin-quizzes.ts index 08e9e60..b992ce2 100644 --- a/apps/server/src/http/admin-quizzes.ts +++ b/apps/server/src/http/admin-quizzes.ts @@ -16,7 +16,8 @@ import { getImportJob, startImport, type Lavalink } from '../quizzes/playlist'; import { applyReview, listReviews, pendingCount, rejectReview, syncReviews } from '../quizzes/review'; import { checkEmbed, type EmbedState } from '../collector/oembed'; import { DuelViewsJob, type ViewsSource } from '../quizzes/duel-views'; -import { songAnswers } from '../answers'; +import { songAnswersBatch } from '../answers'; +import { wikiLookup, type WikiLookup } from '../answers/wiki'; type Env = { Variables: { userId: string | null; role: Role } }; @@ -33,6 +34,8 @@ export interface QuizAdminDeps { lavalinkFetch?: typeof fetch; checkEmbed?: (videoId: string) => Promise; importDelayMs?: number; + /** J-POP 한글 이름 lookup (위키백과); tests inject a fake. */ + wiki?: WikiLookup; videoOf: (url: string) => Promise<{ id: string; author: string; title: string } | string>; onChange?: () => void; /** 이지선다 조회수 자동: where pairs and view counts come from (tests inject a fake) and the pause between songs. */ @@ -235,7 +238,15 @@ export function mountQuizAdmin(app: Hono, d: QuizAdminDeps, h: { actor: (c: const lavalink = d.lavalink !== undefined ? d.lavalink : process.env.LAVALINK_URL && process.env.LAVALINK_PW ? { url: process.env.LAVALINK_URL, password: process.env.LAVALINK_PW } : null; const job = await startImport( - { db: d.db, lavalink, search: d.searchYoutube ?? defaultSearch, embed: d.checkEmbed ?? ((id) => checkEmbed(id)), delayMs: d.importDelayMs, fetchImpl: d.lavalinkFetch }, + { + db: d.db, + lavalink, + search: d.searchYoutube ?? defaultSearch, + embed: d.checkEmbed ?? ((id) => checkEmbed(id)), + delayMs: d.importDelayMs, + fetchImpl: d.lavalinkFetch, + wiki: d.wiki ?? wikiLookup(), + }, h.actor(c), p.data.url, ); @@ -257,11 +268,8 @@ export function mountQuizAdmin(app: Hono, d: QuizAdminDeps, h: { actor: (c: .safeParse(await h.body(c)); if (!p.success) return c.json({ error: '입력 값이 올바르지 않아요.' }, 400); const byKey = d.db.query<{ id: number }, [string]>('SELECT id FROM songs WHERE match_key = ? AND deleted_at IS NULL ORDER BY id LIMIT 1'); - const answers = p.data.songs.map((s) => { - const songId = s.artist ? (byKey.get(songMatchKey(s.title, s.artist))?.id ?? null) : null; - const a = songAnswers(d.db, { ...s, songId }); - return { titleAnswers: a.titleAnswers, artistAnswers: a.artistAnswers }; - }); + const songs = p.data.songs.map((s) => ({ ...s, songId: s.artist ? (byKey.get(songMatchKey(s.title, s.artist))?.id ?? null) : null })); + const answers = (await songAnswersBatch(d.db, songs)).map((a) => ({ titleAnswers: a.titleAnswers, artistAnswers: a.artistAnswers })); return c.json({ answers }); }); diff --git a/apps/server/src/http/admin.ts b/apps/server/src/http/admin.ts index 88c238a..5352c66 100644 --- a/apps/server/src/http/admin.ts +++ b/apps/server/src/http/admin.ts @@ -44,6 +44,8 @@ export interface AdminDeps { lavalink?: { url: string; password: string } | null; lavalinkFetch?: typeof fetch; importDelayMs?: number; + /** J-POP 한글 이름 (위키백과); tests inject a fake. */ + wiki?: import('../answers/wiki').WikiLookup; /** 연령제한 곡's downloaded audio, for the quiz editor's position bar. */ audio?: AudioLinks; /** 이지선다 조회수 자동 (tests inject a fake source). */ @@ -132,6 +134,7 @@ export function createAdminApp(d: AdminDeps) { lavalink: d.lavalink, lavalinkFetch: d.lavalinkFetch, importDelayMs: d.importDelayMs, + wiki: d.wiki, viewsSource: d.viewsSource, viewsDelayMs: d.viewsDelayMs, videoOf, diff --git a/apps/server/src/quizzes/playlist.ts b/apps/server/src/quizzes/playlist.ts index 5a849a9..552268c 100644 --- a/apps/server/src/quizzes/playlist.ts +++ b/apps/server/src/quizzes/playlist.ts @@ -13,7 +13,10 @@ import { isConfident, scoreResult } from '../collector/score'; import type { YtmSong } from '../collector/ytmusic'; import type { EmbedState } from '../collector/oembed'; import { aliasesOf } from '../songs/youtube-review'; -import { songAnswers } from '../answers'; +import { cleanTitle } from '@mq/shared'; +import { songAnswersBatch } from '../answers'; +import type { JaReader } from '../answers/ja-reading'; +import type { WikiLookup } from '../answers/wiki'; export interface LavalinkTrack { identifier: string; @@ -106,6 +109,9 @@ export interface ImportDeps { embed: (videoId: string) => Promise; delayMs?: number; fetchImpl?: typeof fetch; + /** J-POP 한글 이름 (위키백과) and 일본어 읽기 for the answers (docs/13). */ + wiki?: WikiLookup; + read?: JaReader; } /** Starts an import job and returns it at once (rows fill in while it runs). */ @@ -170,11 +176,22 @@ async function runImport(d: ImportDeps, job: ImportJob, tracks: LavalinkTrack[]) } else if (!row.note) row.note = '유튜브에서 못 찾았어요'; if (i < tracks.length - 1) await Bun.sleep(d.delayMs ?? 700); } - // shown title without brackets / feat. / version marks, plus the answers worked out for it (docs/13) - const a = songAnswers(d.db, row); - Object.assign(row, { title: a.title, titleAnswers: a.titleAnswers, artistAnswers: a.artistAnswers }); + // shown title without brackets / feat. / version marks (the answers come at the end, docs/13) + row.title = cleanTitle(row.title).title; job.rows.push(row); job.done = i + 1; } + // answers for every song at once: 한글 발음, 일본어 읽기 (one process), J-POP 한글 이름 (위키백과) + const answers = await songAnswersBatch( + d.db, + job.rows.map((r) => ({ title: r.title, artist: r.artist, songId: r.songId })), + { korean: true, wiki: d.wiki, read: d.read }, + ); + job.rows.forEach((r, i) => { + const a = answers[i]!; + Object.assign(r, { title: a.title, artist: a.artist, titleAnswers: a.titleAnswers, artistAnswers: a.artistAnswers }); + if (a.note) r.note = [r.note, a.note].filter(Boolean).join(' · '); + if (a.note?.includes('확인 필요') && r.status !== 'blocked' && r.status !== 'missing') r.status = 'check'; + }); job.status = 'done'; } diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index fb4d573..b4abf2c 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -44,7 +44,7 @@ export interface StartOptions { /** Quiz editor externals (YouTube search, Lavalink playlists); tests inject fakes. */ /** 유튜브 뮤직 쿠키 상태 for 관리자 → 수집 (index.ts starts the keeper in production). */ cookieStatus?: () => import('./collector/cookie-keeper').CookieStatus; - quizAdmin?: Pick; + quizAdmin?: Pick; } export function startServer(opts: StartOptions) { diff --git a/bun.lock b/bun.lock index 42dceb0..46cbff8 100644 --- a/bun.lock +++ b/bun.lock @@ -16,10 +16,15 @@ "version": "0.0.0", "dependencies": { "@mq/shared": "workspace:*", + "cmu-pronouncing-dictionary": "3.0.0", "hono": "4.13.12", + "kuromoji": "0.1.2", "ulid": "3.0.2", "zod": "4.6.5", }, + "devDependencies": { + "@types/kuromoji": "^0.1.3", + }, }, "apps/web": { "name": "@mq/web", @@ -92,6 +97,10 @@ "@types/bun": ["@types/bun@1.4.2", "", { "dependencies": { "bun-types": "1.4.2" } }, "sha512-GimotNn7+ZV0uVArItBbriZsR1oNf0+WTzPkdcFrzShI7k2norL0uzEaJT8T33dWr7O/c9ZDuAFQrctKCi72oQ=="], + "@types/doublearray": ["@types/doublearray@0.0.32", "", {}, "sha512-HloTru3I3a55runIVqZX1YBQi2L5A4peNQPh33yshzB4ttt1qHCnHPkuhy9Djy/cTx7i5xJvxItKRPCmvnfpGw=="], + + "@types/kuromoji": ["@types/kuromoji@0.1.3", "", { "dependencies": { "@types/doublearray": "*" } }, "sha512-u+YwX6eJj6Fmm0F5qunsyA+X8HSiyRNNE5ON3itD3tERax4meq9tv+S7bjTMXkPjqbdBGUmH2maGDCuEvpODwg=="], + "@types/node": ["@types/node@26.6.5", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-uMzsv9stDdofIZBx4UGh4Z9ZgXnGC9seRDscilBcTHAeOFk67MZg/k9bWVfO+bk4qZmG+l52hRnLwUiH2LnJTg=="], "@types/react": ["@types/react@19.3.0", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-N0rFCuH9YoxG9/m61l9MfpJKfmLOVU0em7ipIz6TRgSSkvReLB9vL85GB+yr8Bs5leqpvg96JSwF4ZS1s4viQg=="], @@ -100,22 +109,30 @@ "@vitejs/plugin-react": ["@vitejs/plugin-react@6.1.1", "", { "dependencies": { "@rolldown/pluginutils": "^1.0.1" }, "peerDependencies": { "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", "babel-plugin-react-compiler": "^1.0.0", "oxc-transform-react": "^0.145.0", "vite": "^8.0.0" }, "optionalPeers": ["@rolldown/plugin-babel", "babel-plugin-react-compiler", "oxc-transform-react"] }, "sha512-yxLaQV9gkhS8ezJqCM6+ndU7mDY6gqAg75NQ+0IjwEI8IYOmQCgkRwHKVSfWXW076DsqMo0Dk+0FK1U+M5RgFw=="], + "async": ["async@2.6.4", "", { "dependencies": { "lodash": "^4.17.14" } }, "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA=="], + "axe-core": ["axe-core@4.13.0", "", {}, "sha512-UzGt8zg7Ny8djbYMhxl2zuEevVa7r2gJjYY5Lwr1xM7+XU2nd6CkIWFTVcCIbAP63vSz71NaVyyuSk9lHKcy0A=="], "bun-types": ["bun-types@1.4.2", "", { "dependencies": { "@types/node": "*" } }, "sha512-bxV1FgK7yBIzjRe5zBozIM4Bem11ZJcCXSrjWRG3YWLt8yFDePu4cLjpebO8OvPeIE9trbyPF4fuj3Cia4Fj3w=="], + "cmu-pronouncing-dictionary": ["cmu-pronouncing-dictionary@3.0.0", "", {}, "sha512-6y5X4OxsYFnHERo6WSVPnLrk53G07MOQ8aUb2z555JgwynpbZsHR0a//WdN6DUcmE1j2auTF2TtL4PpoKze8QQ=="], + "cookie-es": ["cookie-es@3.1.1", "", {}, "sha512-UaXxwISYJPTr9hwQxMFYZ7kNhSXboMXP+Z3TRX6f1/NyaGPfuNUZOWP1pUEb75B2HjfklIYLVRfWiFZJyC6Npg=="], "csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="], "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], + "doublearray": ["doublearray@0.0.2", "", {}, "sha512-aw55FtZzT6AmiamEj2kvmR6BuFqvYgKZUkfQ7teqVRNqD5UE0rw8IeW/3gieHNKQ5sPuDKlljWEn4bzv5+1bHw=="], + "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], "hono": ["hono@4.13.12", "", {}, "sha512-6E2QDAc9Ick9Sq77ZrGS/dk2WUYni91aufTw6LJKpV7w8kW5/GxVUc650FOADOmlwg3K+f7Pun6XlV+pYmW6gw=="], + "kuromoji": ["kuromoji@0.1.2", "", { "dependencies": { "async": "^2.0.1", "doublearray": "0.0.2", "zlibjs": "^0.3.1" } }, "sha512-V0dUf+C2LpcPEXhoHLMAop/bOht16Dyr+mDiIE39yX3vqau7p80De/koFqpiTcL1zzdZlc3xuHZ8u5gjYRfFaQ=="], + "lightningcss": ["lightningcss@1.33.0", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.33.0", "lightningcss-darwin-arm64": "1.33.0", "lightningcss-darwin-x64": "1.33.0", "lightningcss-freebsd-x64": "1.33.0", "lightningcss-linux-arm-gnueabihf": "1.33.0", "lightningcss-linux-arm64-gnu": "1.33.0", "lightningcss-linux-arm64-musl": "1.33.0", "lightningcss-linux-x64-gnu": "1.33.0", "lightningcss-linux-x64-musl": "1.33.0", "lightningcss-win32-arm64-msvc": "1.33.0", "lightningcss-win32-x64-msvc": "1.33.0" } }, "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA=="], "lightningcss-android-arm64": ["lightningcss-android-arm64@1.33.0", "", { "os": "android", "cpu": "arm64" }, "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg=="], @@ -140,6 +157,8 @@ "lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.33.0", "", { "os": "win32", "cpu": "x64" }, "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA=="], + "lodash": ["lodash@4.18.1", "", {}, "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q=="], + "nanoid": ["nanoid@3.3.20", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ=="], "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], @@ -174,6 +193,8 @@ "vite": ["vite@8.3.2", "", { "dependencies": { "lightningcss": "^1.33.0", "picomatch": "^4.0.7", "postcss": "^8.5.28", "rolldown": "~1.2.11", "tinyglobby": "^0.2.17" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "@vitejs/devtools": "^0.7.1", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "@vitejs/devtools", "esbuild", "jiti", "less", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-SQr1x6W5vVSbROg7vsyXIaxK9b0G7zsT68acdWWRmnBUsgDieLCRG+Rep9WdZgcposvv/GSnr4GUUBqB3vXq6w=="], + "zlibjs": ["zlibjs@0.3.1", "", {}, "sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w=="], + "zod": ["zod@4.6.5", "", {}, "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q=="], "zustand": ["zustand@5.0.15", "", { "peerDependencies": { "@types/react": ">=18.0.0", "immer": ">=9.0.6", "react": ">=18.0.0", "use-sync-external-store": ">=1.2.0" }, "optionalPeers": ["@types/react", "immer", "react", "use-sync-external-store"] }, "sha512-MpSEjRiBkA9crSYeOUH32rJC7SVqAbm0Fqcqge/bUi2PPoLcBWKOsG+C8mevmpr8TwXHBVkChbbJiyvkE+i/3A=="], diff --git a/docs/13-title-answers.md b/docs/13-title-answers.md index f54d6e9..f77a21c 100644 --- a/docs/13-title-answers.md +++ b/docs/13-title-answers.md @@ -24,7 +24,13 @@ ## 단계 1. [x] `packages/shared/src/title-clean.ts` cleanTitle / versionKind / videoVersionKind, `cli.ts drop-versions`, 차트·원곡 찾기 규칙 (239c314) 2. [x] 엑셀 양식: 제목/정답 열 나누기(quiz-file.ts), 재생목록 엑셀 두 양식, 파일 정답 자동 채움(`POST /api/admin/quizzes/answers`, 서버 `answers/index.ts`) -3. [ ] 정답 자동 생성: 영어 → 한글 발음(CMUdict + 규칙 + 용례), 일본어 → 원어·읽는 소리(kuromoji + 가나→한글)·위키백과 한글 이름 +3. [x] 정답 자동 생성 `apps/server/src/answers/` + - en-ko.ts: CMU 발음사전(첫 영어 제목 때 읽음, 약 95MB) + 외래어 표기 규칙 + 흔한 표기 단어표(COMMON). 못 읽는 단어가 있으면 그 제목은 발음 없음. 사전에 없는 단어는 로마자(일본어)·두 단어 붙임으로 읽어 봄 + - ja-ko.ts: 가나 → 한글 두 가지(rule 표기법: 요루니 가케루 / common 거센소리: 요루니 카케루), 로마자 → 가나 + - ja-reading.ts + ja-worker.ts: 한자 읽기는 kuromoji를 별도 bun 프로세스로(약 350MB, 서버에 올리지 않음, 묶음마다 한 번, 약 0.3초) + - wiki.ts: 일본어 위키백과 → 한국어 문서 이름(50개씩, 넘겨주기 따라감) + - index.ts songAnswersBatch: `korean` 옵션이면 일본어 제목·가수를 한글 이름으로 보여 줌(위키백과 → 괄호 안 한글 → 읽는 소리 '한글 이름 자동 변환' → '확인 필요') + - 플레이리스트 가져오기·재생목록 엑셀: korean + 위키백과, 파일 정답 채우기: 이름은 그대로 두고 정답만 4. [ ] 적용: 플레이리스트 가져오기·재생목록 엑셀·수집(새 곡)·서버 DB 정리 검토 목록(관리자) 5. [ ] 배포: DB 백업 → `drop-versions --dry-run --keep 7985,9066` 확인 → 실행 → 제목 정리 검토 목록