diff --git a/.dockerignore b/.dockerignore index dcb0c7f..506072b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,7 +2,8 @@ node_modules **/node_modules apps/web/dist test-results -data +data/*.db* +data/raw .git deploy/.env docs diff --git a/.gitignore b/.gitignore index 1ae8f1a..21998a0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ apps/web/dist/ data/*.db* test-results/ deploy/.env +data/raw/ diff --git a/apps/server/src/collector/catalog.ts b/apps/server/src/collector/catalog.ts new file mode 100644 index 0000000..2c45d00 --- /dev/null +++ b/apps/server/src/collector/catalog.ts @@ -0,0 +1,110 @@ +/** Writes into the song catalog (docs/07). Idempotent: re-running an import never duplicates songs. */ +import type { Database } from 'bun:sqlite'; +import { songMatchKey } from '@mq/shared'; + +export interface SongInput { + melonId?: number | null; + title: string; + artists: { name: string; melonArtistId?: number | null }[]; + year?: number | null; + durationSec?: number | null; + peakRank?: number | null; + mvYoutubeId?: string | null; + genres?: string[]; + lyrics?: string | null; + source: string; + ref: string; +} + +export class Catalog { + constructor(private db: Database) {} + + private artistId(name: string, melonArtistId: number | null | undefined): number { + if (melonArtistId) { + const r = this.db.query<{ id: number }, [number]>('SELECT id FROM artists WHERE melon_artist_id = ?').get(melonArtistId); + if (r) return r.id; + return Number(this.db.query('INSERT INTO artists (melon_artist_id, name) VALUES (?, ?)').run(melonArtistId, name).lastInsertRowid); + } + const r = this.db.query<{ id: number }, [string]>('SELECT id FROM artists WHERE name = ? ORDER BY melon_artist_id IS NULL LIMIT 1').get(name); + if (r) return r.id; + return Number(this.db.query('INSERT INTO artists (name) VALUES (?)').run(name).lastInsertRowid); + } + + /** Finds the song by Melon id, else by match key; creates it when new. Returns [id, created]. */ + upsert(s: SongInput, now = Date.now()): [number, boolean] { + return this.db.transaction((): [number, boolean] => { + const artist = s.artists.map((a) => a.name).join(', '); + const key = songMatchKey(s.title, artist); + let row = s.melonId ? this.db.query<{ id: number }, [number]>('SELECT id FROM songs WHERE melon_id = ?').get(s.melonId) : null; + row ??= this.db.query<{ id: number }, [string]>('SELECT id FROM songs WHERE match_key = ? AND deleted_at IS NULL ORDER BY id LIMIT 1').get(key); + let id: number; + let created = false; + if (row) { + id = row.id; + // Fill gaps only: never overwrite what an earlier source or an admin already set. + this.db + .query( + `UPDATE songs SET melon_id = COALESCE(melon_id, ?), year = COALESCE(year, ?), duration_sec = COALESCE(duration_sec, ?), + peak_rank = COALESCE(peak_rank, ?), mv_youtube_id = COALESCE(mv_youtube_id, ?), updated_at = ? WHERE id = ?`, + ) + .run(s.melonId ?? null, s.year ?? null, s.durationSec ?? null, s.peakRank ?? null, s.mvYoutubeId ?? null, now, id); + } else { + id = Number( + this.db + .query( + `INSERT INTO songs (melon_id, title, artist, year, duration_sec, peak_rank, mv_youtube_id, match_key, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .run(s.melonId ?? null, s.title, artist, s.year ?? null, s.durationSec ?? null, s.peakRank ?? null, s.mvYoutubeId ?? null, key, now, now).lastInsertRowid, + ); + created = true; + s.artists.forEach((a) => { + this.db.query('INSERT OR IGNORE INTO song_artists (song_id, artist_id, role) VALUES (?, ?, ?)').run(id, this.artistId(a.name, a.melonArtistId), 'main'); + }); + } + for (const g of s.genres ?? []) this.db.query('INSERT OR IGNORE INTO song_genres (song_id, genre_id, source) VALUES (?, ?, ?)').run(id, g, s.source); + if (s.lyrics) this.db.query('INSERT OR IGNORE INTO lyrics (song_id, text, source, fetched_at) VALUES (?, ?, ?, ?)').run(id, s.lyrics, s.source, now); + this.db.query('INSERT OR IGNORE INTO song_sources (song_id, source, ref) VALUES (?, ?, ?)').run(id, s.source, s.ref); + return [id, created]; + })(); + } + + signal(songId: number, kind: string, value: number, now = Date.now()): void { + this.db + .query('INSERT INTO signals (song_id, kind, value, fetched_at) VALUES (?, ?, ?, ?) ON CONFLICT(song_id, kind) DO UPDATE SET value = excluded.value, fetched_at = excluded.fetched_at') + .run(songId, kind, value, now); + } + + /** Automatic level; an admin-set level for the scope is never overwritten. */ + autoLevel(songId: number, scope: string, level: number): void { + this.db + .query(`INSERT INTO song_levels (song_id, scope, level, source) VALUES (?, ?, ?, 'auto') ON CONFLICT(song_id, scope) DO UPDATE SET level = excluded.level WHERE song_levels.source = 'auto'`) + .run(songId, scope, level); + } +} + +export class Job { + readonly id: number; + constructor( + private db: Database, + kind: string, + total = 0, + ) { + // Only one runner per kind: a row left 'running' by a killed process is stale. + this.db.query("UPDATE collect_jobs SET status = 'stopped', finished_at = ? WHERE kind = ? AND status = 'running'").run(Date.now(), kind); + this.id = Number(this.db.query("INSERT INTO collect_jobs (kind, status, total, started_at) VALUES (?, 'running', ?, ?)").run(kind, total, Date.now()).lastInsertRowid); + } + progress(done: number, total?: number): void { + this.db.query('UPDATE collect_jobs SET done = ?, total = COALESCE(?, total) WHERE id = ?').run(done, total ?? null, this.id); + } + log(line: string): void { + console.log(line); + // keep the last ~20KB of log + this.db.query("UPDATE collect_jobs SET log = substr(log || ? || char(10), -20000) WHERE id = ?").run(`${new Date().toISOString().slice(11, 19)} ${line}`, this.id); + } + finish(status: 'done' | 'failed' | 'stopped'): void { + this.db.query('UPDATE collect_jobs SET status = ?, finished_at = ? WHERE id = ?').run(status, Date.now(), this.id); + } +} + +export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); diff --git a/apps/server/src/collector/cli.ts b/apps/server/src/collector/cli.ts new file mode 100644 index 0000000..3d6e7f4 --- /dev/null +++ b/apps/server/src/collector/cli.ts @@ -0,0 +1,46 @@ +/** + * Song catalog collector (docs/07 처리 순서). Runs next to the app on the same DB (WAL): + * bun apps/server/src/collector/cli.ts import-norabose [--limit N] + * bun apps/server/src/collector/cli.ts import-sta + * bun apps/server/src/collector/cli.ts match-youtube [--limit N] [--delay ms] [--require-cookie] (default pace: 2s apart; cron caps a day at 3000) + * bun apps/server/src/collector/cli.ts stats + * In the container: docker exec music_quiz-app-1 bun apps/server/src/collector/cli.ts … + */ +import { join } from 'node:path'; +import { openDb } from '../db'; +import { importNorabose } from './norabose'; +import { importSta } from './sta'; +import { matchYoutube } from './match-youtube'; +import { pickSearcher } from './ytmusic'; +import { catalogStats } from '../songs/store'; + +const [cmd, ...rest] = process.argv.slice(2); +const flag = (name: string) => { + const i = rest.indexOf(`--${name}`); + return i >= 0 ? Number(rest[i + 1]) : undefined; +}; +const dbPath = process.env.DB_PATH ?? './data/app.db'; +const rawDir = process.env.RAW_DIR ?? join(dbPath, '..', 'raw'); +const db = openDb(dbPath); + +switch (cmd) { + case 'import-norabose': + console.log(await importNorabose(db, rawDir, { limit: flag('limit') })); + break; + case 'import-sta': + console.log(await importSta(db, process.env.STA_SEED ?? join(import.meta.dir, '../../../../data/seed/sta_songs.json'))); + break; + case 'match-youtube': { + const searcher = await pickSearcher(process.env.YT_COOKIE_PATH ?? '/home/claude/.config/music_quiz/ytm_cookie.txt', console.log, rest.includes('--require-cookie')); + console.log(`search: ${searcher.kind}`); + console.log(await matchYoutube(db, searcher, { limit: flag('limit'), delayMs: flag('delay'), source: searcher.kind })); + break; + } + case 'stats': + console.log(JSON.stringify(catalogStats(db), null, 1)); + break; + default: + console.log('usage: cli.ts import-norabose|import-sta|match-youtube|stats'); + process.exitCode = 1; +} +db.close(); diff --git a/apps/server/src/collector/collector.test.ts b/apps/server/src/collector/collector.test.ts new file mode 100644 index 0000000..c9c32f1 --- /dev/null +++ b/apps/server/src/collector/collector.test.ts @@ -0,0 +1,113 @@ +import { expect, test } from 'bun:test'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { openDb } from '../db'; +import { Catalog } from './catalog'; +import { genresOf, provisionalLevel } from './norabose'; +import { importSta } from './sta'; +import { matchYoutube } from './match-youtube'; +import { isConfident, scoreResult } from './score'; +import { parseDuration, parseSearch, type YtmSong } from './ytmusic'; +import { catalogStats, listSongs } from '../songs/store'; +import { tempDbPath } from '../test-utils'; + +test('norabose genre bits and provisional levels', () => { + expect(genresOf(1)).toEqual(['ballad', 'kpop']); + expect(genresOf(512 | 2).sort()).toEqual(['dance', 'pop']); + expect(genresOf(8192)).toEqual(['jpop']); + expect(provisionalLevel(0b00001, 3)).toBe(1); + expect(provisionalLevel(0b00001, 40)).toBe(2); + expect(provisionalLevel(0b10100, 50)).toBe(6); + expect(provisionalLevel(0, 1)).toBeNull(); +}); + +test('YouTube Music search parsing', () => { + const row = (videoId: string, title: string, meta: string) => ({ + musicResponsiveListItemRenderer: { + playlistItemData: { videoId }, + flexColumns: [ + { musicResponsiveListItemFlexColumnRenderer: { text: { runs: [{ text: title }] } } }, + { musicResponsiveListItemFlexColumnRenderer: { text: { runs: meta.split(/( • )/).map((text) => ({ text })) } } }, + ], + }, + }); + const r = parseSearch({ contents: [{ x: [row('Wu5FY8J6P8Q', 'Love wins all', '아이유 • The Winning • 4:32'), row('aaaaaaaaaaa', 'no meta', '')] }] }); + expect(r[0]).toEqual({ videoId: 'Wu5FY8J6P8Q', title: 'Love wins all', artists: '아이유', album: 'The Winning', durationSec: 272 }); + expect(parseDuration('1:02:03')).toBe(3723); +}); + +test('scoring: title+artist confident, MR/cover not, wrong length not', () => { + const song = { title: '광 (狂)', artist: 'SG워너비', durationSec: 237 }; + const yt = (title: string, artists: string, durationSec: number): YtmSong => ({ videoId: 'x', title, artists, album: null, durationSec }); + expect(isConfident(scoreResult(song, yt('광 (狂)', 'SG워너비', 237), 0))).toBe(true); + expect(isConfident(scoreResult(song, yt('광', 'SG Wannabe', 236), 1))).toBe(true); // title + exact length + expect(isConfident(scoreResult(song, yt('광 (Inst.)', 'SG워너비', 237), 0))).toBe(false); + expect(isConfident(scoreResult(song, yt('광', 'SG워너비', 300), 0))).toBe(false); + expect(isConfident(scoreResult(song, yt('다른 노래', 'SG워너비', 237), 0))).toBe(false); +}); + +function freshDb() { + return openDb(tempDbPath()); +} + +test('upsert merges the same song from two sources and never duplicates', () => { + const db = freshDb(); + const c = new Catalog(db); + const [a, newA] = c.upsert({ melonId: 100, title: '사건의 지평선', artists: [{ name: '윤하', melonArtistId: 7 }], year: 2022, genres: ['ballad', 'kpop'], source: 'norabose', ref: '100' }); + const [b, newB] = c.upsert({ title: '사건의지평선', artists: [{ name: '윤하' }], year: 2026, genres: ['kpop'], source: 'sta', ref: '5' }); + expect(newA).toBe(true); + expect(newB).toBe(false); + expect(b).toBe(a); + const s = db.query<{ year: number }, []>('SELECT year FROM songs').get()!; + expect(s.year).toBe(2022); // the second source only fills gaps + expect(catalogStats(db).sources).toEqual({ norabose: 1, sta: 1 }); + c.autoLevel(a, 'all', 3); + db.query("UPDATE song_levels SET level = 9, source = 'admin' WHERE song_id = ?").run(a); + c.autoLevel(a, 'all', 4); + expect(db.query<{ level: number }, []>('SELECT level FROM song_levels').get()!.level).toBe(9); // admin level kept +}); + +test('sta import skips map-maker years and merges duplicates', async () => { + const db = freshDb(); + new Catalog(db).upsert({ melonId: 1, title: 'Candy Man', artists: [{ name: '브라운아이드걸스' }], year: 2009, source: 'norabose', ref: '1' }); + const dir = mkdtempSync(join(process.env.TMPDIR ?? '/tmp', 'mq-sta-')); + const seed = join(dir, 'sta.json'); + writeFileSync(seed, JSON.stringify({ songs: [ + { id: 1, title: 'Candy Man', artist: '브라운아이드걸스', year: 2009, found_in: ['a'] }, + { id: 2, title: 'Vitamin ME', artist: '프로미스나인', year: new Date().getFullYear(), found_in: ['a'] }, + ] })); + expect(await importSta(db, seed)).toEqual({ total: 2, created: 1 }); + const v = db.query<{ year: number | null }, [string]>('SELECT year FROM songs WHERE title = ?').get('Vitamin ME')!; + expect(v.year).toBeNull(); +}); + +test('YouTube matching: confident → matched, unsure → candidate fallback, noembed skipped', async () => { + const db = freshDb(); + const c = new Catalog(db); + const [s1] = c.upsert({ title: 'Love wins all', artists: [{ name: '아이유' }], durationSec: 272, source: 't', ref: '1' }); + const [s2] = c.upsert({ title: '한사람을 위해', artists: [{ name: '안재모' }], durationSec: 293, source: 't', ref: '2' }); + db.query("INSERT INTO youtube_candidates (song_id, video_id, score, source) VALUES (?, 'fanvideo001', 0, 'norabose')").run(s2); + const [s3] = c.upsert({ title: '없는노래', artists: [{ name: '누구' }], source: 't', ref: '3' }); + const results: Record = { + '아이유 Love wins all': [ + { videoId: 'blockedvid1', title: 'Love wins all', artists: '아이유', album: null, durationSec: 272 }, + { videoId: 'Wu5FY8J6P8Q', title: 'Love wins all', artists: '아이유', album: 'The Winning', durationSec: 272 }, + ], + '안재모 한사람을 위해': [{ videoId: 'othersong01', title: '다른 노래', artists: '안재모', album: null, durationSec: 200 }], + '누구 없는노래': [], + }; + const stats = await matchYoutube( + db, + { searchSongs: async (q) => results[q] ?? [] }, + { delayMs: 0 }, + async (id) => (id === 'blockedvid1' ? { state: 'noembed' } : { state: 'ok', author: id === 'Wu5FY8J6P8Q' ? 'IU - Topic' : 'fan', title: id }), + ); + expect(stats).toEqual({ matched: 1, candidate: 1, none: 1 }); + const row = (id: number) => db.query<{ youtube_id: string | null; youtube_status: string; youtube_channel: string | null }, [number]>('SELECT youtube_id, youtube_status, youtube_channel FROM songs WHERE id = ?').get(id)!; + expect(row(s1)).toEqual({ youtube_id: 'Wu5FY8J6P8Q', youtube_status: 'matched', youtube_channel: 'IU - Topic' }); + expect(row(s2)).toMatchObject({ youtube_id: 'fanvideo001', youtube_status: 'candidate' }); + expect(row(s3).youtube_status).toBe('none'); + // Re-run skips songs tried recently. + expect(await matchYoutube(db, { searchSongs: async () => { throw new Error('should not search'); } }, { delayMs: 0 })).toEqual({ matched: 0, candidate: 0, none: 0 }); + expect(listSongs(db, { youtube: 'matched' }).total).toBe(1); +}); diff --git a/apps/server/src/collector/match-youtube.ts b/apps/server/src/collector/match-youtube.ts new file mode 100644 index 0000000..9faab99 --- /dev/null +++ b/apps/server/src/collector/match-youtube.ts @@ -0,0 +1,112 @@ +/** + * Finds the official audio (Topic / Art Track) for each song via YouTube Music song search (docs/08 §1), + * checks it can be embedded (IFrame playback, docs/12 Q1) and records candidates for the admin. + * Resumable: songs tried in the last `retryDays` are skipped. Stops on cookie problems. + */ +import type { Database } from 'bun:sqlite'; +import { Job, sleep } from './catalog'; +import { checkEmbed } from './oembed'; +import { isConfident, scoreResult, type Scored } from './score'; +import { CookieError, type YtMusic, type YtmSong } from './ytmusic'; + +interface Row { + id: number; + title: string; + artist: string; + duration_sec: number | null; +} + +export interface MatchOptions { + limit?: number; + delayMs?: number; + retryDays?: number; + /** Where results come from: recorded on songs and candidates. */ + source?: 'ytmusic' | 'youtube'; + log?: (s: string) => void; +} + +export async function matchYoutube(db: Database, ytm: Pick, opts: MatchOptions = {}, embed = checkEmbed) { + const retryBefore = Date.now() - (opts.retryDays ?? 7) * 86400_000; + // Tries are remembered per search method: songs the weaker fallback could not settle are retried once the cookie works. + const triedKind = `yt_tried_${opts.source ?? 'ytmusic'}`; + const rows = db + .query( + `SELECT s.id, s.title, s.artist, s.duration_sec FROM songs s + LEFT JOIN signals t ON t.song_id = s.id AND t.kind = ? + WHERE s.deleted_at IS NULL AND s.youtube_status IN ('none','candidate') AND COALESCE(s.youtube_source, '') <> 'admin' + AND (t.value IS NULL OR t.value < ?) + ORDER BY s.id LIMIT ?`, + ) + .all(triedKind, retryBefore, opts.limit ?? 1_000_000); + const job = new Job(db, 'match-youtube', rows.length); + const stats = { matched: 0, candidate: 0, none: 0 }; + const src = opts.source ?? 'ytmusic'; + const setVideo = (id: number, status: 'matched' | 'candidate', v: { videoId: string; author: string; title: string }, source: string) => + db + .query( + `UPDATE songs SET youtube_id = ?, youtube_status = ?, youtube_source = ?, youtube_channel = ?, youtube_title = ?, embeddable = 1, + youtube_checked_at = ?, updated_at = ? WHERE id = ?`, + ) + .run(v.videoId, status, source, v.author, v.title, Date.now(), Date.now(), id); + try { + for (const [i, s] of rows.entries()) { + let results: YtmSong[]; + const query = `${s.artist.split(', ')[0]} ${s.title}`; + try { + results = await ytm.searchSongs(query); + } catch (e) { + if (e instanceof CookieError) throw e; + job.log(`search failed (${(e as Error).message}), waiting 60s`); + await sleep(60_000); + results = await ytm.searchSongs(query); + } + const scored: Scored[] = results.slice(0, 8).map((r, rank) => scoreResult({ title: s.title, artist: s.artist, durationSec: s.duration_sec }, r, rank)); + scored.sort((a, b) => b.score - a.score); + for (const c of scored.slice(0, 5)) { + db.query( + `INSERT INTO youtube_candidates (song_id, video_id, title, channel, duration_sec, score, source) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(song_id, video_id) DO UPDATE SET title = excluded.title, channel = excluded.channel, duration_sec = excluded.duration_sec, score = excluded.score`, + ).run(s.id, c.videoId, c.title, c.artists, c.durationSec, c.score, src); + } + let done = false; + for (const c of scored.filter(isConfident)) { + const e = await embed(c.videoId); + if (e.state === 'ok') { + setVideo(s.id, 'matched', { videoId: c.videoId, author: e.author, title: e.title }, src); + if (!s.duration_sec && c.durationSec) db.query('UPDATE songs SET duration_sec = ? WHERE id = ?').run(c.durationSec, s.id); + stats.matched++; + done = true; + break; + } + } + if (!done) { + // Not sure: keep the best playable guess as a candidate for the admin (title match first, then norabose's video). + const fallbacks: { videoId: string; source: string }[] = [ + ...scored.filter((c) => c.titleOk).slice(0, 2).map((c) => ({ videoId: c.videoId, source: src })), + ...db.query<{ video_id: string }, [number]>("SELECT video_id FROM youtube_candidates WHERE song_id = ? AND source = 'norabose'").all(s.id).map((r) => ({ videoId: r.video_id, source: 'norabose' })), + ]; + for (const f of fallbacks) { + const e = await embed(f.videoId); + if (e.state === 'ok') { + setVideo(s.id, 'candidate', { videoId: f.videoId, author: e.author, title: e.title }, f.source); + stats.candidate++; + done = true; + break; + } + } + if (!done) stats.none++; + } + db.query('INSERT INTO signals (song_id, kind, value, fetched_at) VALUES (?, ?, ?, ?) ON CONFLICT(song_id, kind) DO UPDATE SET value = excluded.value, fetched_at = excluded.fetched_at').run(s.id, triedKind, Date.now(), Date.now()); + job.progress(i + 1); + if ((i + 1) % 50 === 0) job.log(`youtube ${i + 1}/${rows.length} matched ${stats.matched} candidate ${stats.candidate} none ${stats.none}`); + await sleep(opts.delayMs ?? 2000); + } + job.log(`youtube done ${rows.length}: matched ${stats.matched} candidate ${stats.candidate} none ${stats.none}`); + job.finish('done'); + } catch (e) { + job.log(`stopped: ${(e as Error).message}`); + job.finish(e instanceof CookieError ? 'stopped' : 'failed'); + throw e; + } + return stats; +} diff --git a/apps/server/src/collector/norabose.ts b/apps/server/src/collector/norabose.ts new file mode 100644 index 0000000..393c5b0 --- /dev/null +++ b/apps/server/src/collector/norabose.ts @@ -0,0 +1,148 @@ +/** + * norabose.com import (docs/07, Q11 ★예): quiz index (year, peak, genre mask, level mask) + song details in batches of 50. + * Raw responses are cached under rawDir so a re-run does not fetch again. 1.5s between requests. + */ +import { existsSync, mkdirSync } from 'node:fs'; +import { join } from 'node:path'; +import type { Database } from 'bun:sqlite'; +import { Catalog, Job, sleep } from './catalog'; + +const BASE = 'https://norabose.com'; +const UA = 'Mozilla/5.0 (music_quiz catalog; contact via mquiz.tkrmagid.kr)'; + +/** norabose genre bits (their bundle, genres-*.js) → our genre ids (docs/04 장르). */ +const GENRE_BITS: [number, string | null][] = [ + [1, 'ballad'], [2, 'dance'], [4, 'hiphop'], [8, 'rock'], [16, 'rnb'], [32, 'ost'], [64, 'folk'], [128, 'trot'], + [256, 'indie'], [512, 'pop'], [1024, 'ost'], [2048, 'electronic'], [4096, 'kids'], [8192, 'jpop'], [16384, 'anime'], + [32768, 'jazz'], [65536, null], [131072, 'kids'], [262144, 'classical'], [524288, 'jazz'], [1048576, 'jazz'], + [2097152, 'classical'], [4194304, 'anime'], [8388608, 'classical'], [16777216, 'classical'], [33554432, null], +]; + +export function genresOf(mask: number): string[] { + const out = new Set(); + for (const [bit, id] of GENRE_BITS) if (mask & bit && id) out.add(id); + // Region: their data is the Melon chart; POP / J-POP bits mark foreign songs, everything else is Korean. + if (!out.has('pop') && !out.has('jpop')) out.add('kpop'); + return [...out]; +} + +/** + * Provisional 1~10 level (real calculation is stage 9): norabose has 5 levels (bit mask, a song can be in several). + * Lowest level L → 2L-1 for weekly top-10 songs, else 2L. + */ +export function provisionalLevel(levelMask: number, peak: number | null): number | null { + for (let l = 1; l <= 5; l++) if (levelMask & (1 << (l - 1))) return Math.min(10, 2 * l - (peak !== null && peak <= 10 ? 1 : 0)); + return null; +} + +interface NbSong { + id: number; + title: string; + genre: number; + length: number | null; + youtube_id: string | null; + mv_id: string | null; +} +interface NbBatch { + songs: NbSong[]; + song_artists: { song_id: number; artist_id: number; role_code: number }[]; + artist_names: { artist_id: number; name: string }[]; + lyrics: { song_id: number; text: string }[]; +} + +async function getJson(url: string): Promise { + for (let attempt = 0; ; attempt++) { + const res = await fetch(url, { headers: { 'user-agent': UA }, signal: AbortSignal.timeout(20_000) }); + if (res.ok) return (await res.json()) as T; + if (attempt >= 1 || res.status === 404) throw new Error(`${url} → ${res.status}`); + await sleep(5000 * (attempt + 1)); + } +} + +/** Their API answers 500 for a batch that holds a broken song: split the batch until only the broken ids are left out. */ +async function fetchBatch(ids: number[], delayMs: number, skipped: number[]): Promise { + try { + const b = await getJson(`${BASE}/api/norabose/songs/${ids.join(',')}`); + await sleep(delayMs); + return b; + } catch (e) { + if (!/→ 5\d\d$/.test((e as Error).message)) throw e; + await sleep(delayMs); + if (ids.length === 1) { + skipped.push(ids[0]!); + return { songs: [], song_artists: [], artist_names: [], lyrics: [] }; + } + const half = Math.ceil(ids.length / 2); + const [x, y] = [await fetchBatch(ids.slice(0, half), delayMs, skipped), await fetchBatch(ids.slice(half), delayMs, skipped)]; + return { songs: [...x.songs, ...y.songs], song_artists: [...x.song_artists, ...y.song_artists], artist_names: [...x.artist_names, ...y.artist_names], lyrics: [...x.lyrics, ...y.lyrics] }; + } +} + +export async function importNorabose(db: Database, rawDir: string, opts: { limit?: number; delayMs?: number } = {}) { + const dir = join(rawDir, 'norabose'); + mkdirSync(dir, { recursive: true }); + const indexPath = join(dir, 'quiz-index.json'); + if (!existsSync(indexPath)) await Bun.write(indexPath, JSON.stringify(await getJson(`${BASE}/data/quiz-index.json`))); + const index = (await Bun.file(indexPath).json()) as { songs: [number, number, number, number, number, string][] }; + const rows = opts.limit ? index.songs.slice(0, opts.limit) : index.songs; + const meta = new Map(rows.map((r) => [r[0], { year: r[1], peak: r[2], genreMask: r[3], levelMask: r[4] }])); + const ids = [...meta.keys()]; + const job = new Job(db, 'import-norabose', ids.length); + const catalog = new Catalog(db); + let created = 0; + let done = 0; + try { + for (let i = 0; i < ids.length; i += 50) { + const chunk = ids.slice(i, i + 50); + const file = join(dir, `songs-${chunk[0]}-${chunk.length}.json`); + if (!existsSync(file)) { + const skipped: number[] = []; + await Bun.write(file, JSON.stringify(await fetchBatch(chunk, opts.delayMs ?? 1500, skipped))); + if (skipped.length) job.log(`norabose: skipped broken ids ${skipped.join(',')}`); + } + const b = (await Bun.file(file).json()) as NbBatch; + const names = new Map(b.artist_names.map((a) => [a.artist_id, a.name])); + const lyrics = new Map(b.lyrics.map((l) => [l.song_id, l.text.replace(/\\n/g, '\n')])); + db.transaction(() => { + for (const s of b.songs) { + const m = meta.get(s.id); + if (!m) continue; + const singers = b.song_artists.filter((a) => a.song_id === s.id && a.role_code & 1).map((a) => ({ name: names.get(a.artist_id) ?? '?', melonArtistId: a.artist_id })); + if (!singers.length) continue; + const [songId, isNew] = catalog.upsert({ + melonId: s.id, + title: s.title, + artists: singers, + year: m.year, + durationSec: s.length, + peakRank: m.peak, + mvYoutubeId: s.mv_id, + genres: genresOf(m.genreMask | s.genre), + lyrics: lyrics.get(s.id) ?? null, + source: 'norabose', + ref: String(s.id), + }); + if (isNew) created++; + // Their youtube_id is often a fan lyric video: keep it only as a fallback candidate (docs/08). + if (s.youtube_id) { + db.query("INSERT OR IGNORE INTO youtube_candidates (song_id, video_id, score, source) VALUES (?, ?, 0, 'norabose')").run(songId, s.youtube_id); + } + catalog.signal(songId, 'norabose_level_mask', m.levelMask); + catalog.signal(songId, 'norabose_peak', m.peak); + const lv = provisionalLevel(m.levelMask, m.peak); + if (lv) catalog.autoLevel(songId, 'all', lv); + } + })(); + done += chunk.length; + job.progress(done); + if ((i / 50) % 20 === 0) job.log(`norabose ${done}/${ids.length}, new ${created}`); + } + job.log(`norabose done: ${done} songs, new ${created}`); + job.finish('done'); + } catch (e) { + job.log(`failed: ${(e as Error).message}`); + job.finish('failed'); + throw e; + } + return { total: ids.length, created }; +} diff --git a/apps/server/src/collector/oembed.ts b/apps/server/src/collector/oembed.ts new file mode 100644 index 0000000..864d3c1 --- /dev/null +++ b/apps/server/src/collector/oembed.ts @@ -0,0 +1,13 @@ +/** Embeddability check (docs/12 Q1: IFrame playback needs embeddable videos). oEmbed: 200 ok, 401 embedding disabled, 404/400 gone. */ +export type EmbedState = { state: 'ok'; author: string; title: string } | { state: 'noembed' } | { state: 'dead' } | { state: 'error'; status: number }; + +export async function checkEmbed(videoId: string, fetchImpl: typeof fetch = fetch): Promise { + const res = await fetchImpl(`https://www.youtube.com/oembed?format=json&url=${encodeURIComponent(`https://youtu.be/${videoId}`)}`, { signal: AbortSignal.timeout(10_000) }); + if (res.ok) { + const j = (await res.json()) as { author_name: string; title: string }; + return { state: 'ok', author: j.author_name, title: j.title }; + } + if (res.status === 401 || res.status === 403) return { state: 'noembed' }; + if (res.status === 404 || res.status === 400) return { state: 'dead' }; + return { state: 'error', status: res.status }; +} diff --git a/apps/server/src/collector/score.ts b/apps/server/src/collector/score.ts new file mode 100644 index 0000000..c6f4053 --- /dev/null +++ b/apps/server/src/collector/score.ts @@ -0,0 +1,36 @@ +/** Scores YouTube Music results against a song (docs/08 매칭 순서 4~5). */ +import { artistKeys, keysOverlap, titleKeys } from '@mq/shared'; +import type { YtmSong } from './ytmusic'; + +const JUNK = /\b(inst|instrumental|mr|karaoke|cover|remix|sped ?up|slowed|live|ver\.|version|acoustic)\b|노래방|커버|라이브|반주/i; + +export interface Scored extends YtmSong { + score: number; + titleOk: boolean; + artistOk: boolean; +} + +export function scoreResult(song: { title: string; artist: string; durationSec: number | null }, r: YtmSong, rank: number): Scored { + const titleOk = keysOverlap(titleKeys(song.title), titleKeys(r.title)); + const artistOk = keysOverlap(artistKeys(song.artist), artistKeys(r.artists)); + let score = 0; + if (titleOk) score += 50; + if (artistOk) score += 30; + if (song.durationSec && r.durationSec) { + const d = Math.abs(song.durationSec - r.durationSec); + if (d <= 3) score += 20; + else if (d <= 8) score += 10; + else if (d > 20) score -= 30; + } + if (rank === 0) score += 5; + // Plain YouTube fallback: only official Topic art tracks are trusted; MVs, lyric videos and fan uploads are scored down. + if (r.topic === false) score -= 25; + // Junk words only count when the original title doesn't have them (a real "Remix" song stays fine). + if (JUNK.test(r.title) && !JUNK.test(song.title)) score -= 40; + return { ...r, score, titleOk, artistOk }; +} + +/** Auto-confirm: title match plus artist (80) or exact length (70); a far-off length or junk words drop it below. */ +export function isConfident(s: Scored): boolean { + return s.titleOk && s.score >= 70; +} diff --git a/apps/server/src/collector/sta.ts b/apps/server/src/collector/sta.ts new file mode 100644 index 0000000..4aa6c4d --- /dev/null +++ b/apps/server/src/collector/sta.ts @@ -0,0 +1,36 @@ +/** Starcraft "노래 맞추기" map songs (data/seed/sta_songs.json, docs/07). Years there are unreliable: used only to fill gaps. */ +import type { Database } from 'bun:sqlite'; +import { Catalog, Job } from './catalog'; + +interface StaSong { + id: number; + title: string; + artist: string; + year?: number; + found_in: string[]; +} + +export async function importSta(db: Database, seedPath: string) { + const data = (await Bun.file(seedPath).json()) as { songs: StaSong[] }; + const job = new Job(db, 'import-sta', data.songs.length); + const catalog = new Catalog(db); + const thisYear = new Date().getFullYear(); + let created = 0; + for (const [i, s] of data.songs.entries()) { + const [, isNew] = catalog.upsert({ + title: s.title.trim(), + artists: [{ name: s.artist.trim() }], + // 26 songs say "2026" (map maker's date, not the release): only take years before this year. + year: s.year && s.year < thisYear ? s.year : null, + genres: ['kpop'], + source: 'sta', + ref: String(s.id), + }); + if (isNew) created++; + if (i % 100 === 0) job.progress(i); + } + job.progress(data.songs.length); + job.log(`sta done: ${data.songs.length} songs, new ${created} (rest merged into existing songs)`); + job.finish('done'); + return { total: data.songs.length, created }; +} diff --git a/apps/server/src/collector/ytmusic.ts b/apps/server/src/collector/ytmusic.ts new file mode 100644 index 0000000..98da222 --- /dev/null +++ b/apps/server/src/collector/ytmusic.ts @@ -0,0 +1,149 @@ +/** + * YouTube Music "노래" search with the user's cookie (docs/08 §1, docs/12 쿠키 — song search only). + * Same approach as music_bot_v2: WEB_REMIX client + SAPISIDHASH. + */ +export interface YtmSong { + videoId: string; + title: string; + artists: string; + album: string | null; + durationSec: number | null; + /** Only set by the plain YouTube fallback: true when the channel is an official "… - Topic" art-track channel. */ + topic?: boolean; +} + +export class CookieError extends Error {} + +const ORIGIN = 'https://music.youtube.com'; +/** "Songs" filter of the search tab. */ +const SONGS_PARAMS = 'EgWKAQIIAWoKEAoQCRADEAQQBQ%3D%3D'; + +export function parseDuration(s: string): number | null { + const m = /^(?:(\d+):)?(\d{1,2}):(\d{2})$/.exec(s.trim()); + if (!m) return null; + return Number(m[1] ?? 0) * 3600 + Number(m[2]) * 60 + Number(m[3]); +} + +/** Pulls song rows out of a search response (musicResponsiveListItemRenderer). */ +export function parseSearch(json: unknown): YtmSong[] { + const out: YtmSong[] = []; + const walk = (v: unknown) => { + if (!v || typeof v !== 'object') return; + if (Array.isArray(v)) return v.forEach(walk); + const r = (v as Record).musicResponsiveListItemRenderer; + if (r) { + const videoId = r.playlistItemData?.videoId ?? r.overlay?.musicItemThumbnailOverlayRenderer?.content?.musicPlayButtonRenderer?.playNavigationEndpoint?.watchEndpoint?.videoId; + const cols: string[] = (r.flexColumns ?? []).map((c: any) => (c.musicResponsiveListItemFlexColumnRenderer?.text?.runs ?? []).map((x: any) => x.text).join('')); + if (videoId && cols[0]) { + // second column: "아티스트 • 앨범 • 3:15" (songs filter) + const parts = (cols[1] ?? '').split(' • '); + const last = parts[parts.length - 1] ?? ''; + const dur = parseDuration(last); + out.push({ videoId, title: cols[0], artists: parts[0] ?? '', album: parts.length >= 3 ? parts[1]! : null, durationSec: dur }); + } + return; + } + for (const x of Object.values(v)) walk(x); + }; + walk(json); + return out; +} + +export class YtMusic { + constructor( + private cookie: string, + private fetchImpl: typeof fetch = fetch, + ) {} + + static async fromFile(path: string): Promise { + const cookie = (await Bun.file(path).text()).trim(); + if (!/(?:^|; )SAPISID=/.test(cookie)) throw new CookieError('쿠키에 SAPISID가 없어요.'); + return new YtMusic(cookie); + } + + async searchSongs(query: string): Promise { + const sapisid = /(?:^|; )SAPISID=([^;]+)/.exec(this.cookie)![1]!; + const ts = Math.floor(Date.now() / 1000); + const hash = new Bun.CryptoHasher('sha1').update(`${ts} ${sapisid} ${ORIGIN}`).digest('hex'); + const res = await this.fetchImpl(`${ORIGIN}/youtubei/v1/search?prettyPrint=false`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + cookie: this.cookie, + origin: ORIGIN, + 'x-origin': ORIGIN, + authorization: `SAPISIDHASH ${ts}_${hash}`, + 'x-goog-authuser': '0', + 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0 Safari/537.36', + }, + body: JSON.stringify({ context: { client: { clientName: 'WEB_REMIX', clientVersion: '1.20251015.01.00', hl: 'ko', gl: 'KR' } }, query, params: decodeURIComponent(SONGS_PARAMS) }), + signal: AbortSignal.timeout(15_000), + }); + if (res.status === 401 || res.status === 403) throw new CookieError(`유튜브 뮤직 인증 실패(${res.status}) — 쿠키 갱신 필요`); + if (res.status === 429) throw new Error('rate limited (429)'); + if (!res.ok) throw new Error(`ytmusic search ${res.status}`); + const json: any = await res.json(); + const loggedIn = JSON.stringify(json?.responseContext?.serviceTrackingParams ?? []).includes('"logged_in","value":"1"'); + if (!loggedIn) throw new CookieError('유튜브 뮤직 로그인이 풀렸어요 — 쿠키 갱신 필요'); + return parseSearch(json); + } +} + +/** + * Fallback without login (docs/08 매칭 순서 2): plain YouTube search "{artist} {title} topic", parsed from videoRenderer. + * Weaker than the 노래 tab, so non-Topic results are marked and scored down (score.ts). + */ +export function parseWebSearch(json: unknown): YtmSong[] { + const out: YtmSong[] = []; + const walk = (v: unknown) => { + if (!v || typeof v !== 'object') return; + if (Array.isArray(v)) return v.forEach(walk); + const r = (v as Record).videoRenderer; + if (r?.videoId) { + const channel: string = r.ownerText?.runs?.[0]?.text ?? ''; + const topic = / - Topic$/.test(channel); + out.push({ + videoId: r.videoId, + title: (r.title?.runs ?? []).map((x: any) => x.text).join(''), + artists: topic ? channel.replace(/ - Topic$/, '') : channel, + album: null, + durationSec: r.lengthText?.simpleText ? parseDuration(r.lengthText.simpleText) : null, + topic, + }); + return; + } + for (const x of Object.values(v)) walk(x); + }; + walk(json); + return out; +} + +export class YtWeb { + constructor(private fetchImpl: typeof fetch = fetch) {} + + async searchSongs(query: string): Promise { + const res = await this.fetchImpl('https://www.youtube.com/youtubei/v1/search?prettyPrint=false', { + method: 'POST', + headers: { 'content-type': 'application/json', 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0 Safari/537.36' }, + body: JSON.stringify({ context: { client: { clientName: 'WEB', clientVersion: '2.20251015.01.00', hl: 'ko', gl: 'KR' } }, query: `${query} topic` }), + signal: AbortSignal.timeout(15_000), + }); + if (res.status === 429) throw new Error('rate limited (429)'); + if (!res.ok) throw new Error(`youtube search ${res.status}`); + return parseWebSearch(await res.json()); + } +} + +/** YouTube Music with the cookie when it is still logged in, else the plain YouTube fallback. */ +export async function pickSearcher(cookiePath: string, log: (s: string) => void, requireCookie = false): Promise<{ searchSongs: (q: string) => Promise; kind: 'ytmusic' | 'youtube' }> { + try { + const ytm = await YtMusic.fromFile(cookiePath); + await ytm.searchSongs('아이유 좋은 날'); + return { searchSongs: (q) => ytm.searchSongs(q), kind: 'ytmusic' }; + } catch (e) { + if (!(e instanceof CookieError) || requireCookie) throw e; + log(`유튜브 뮤직 쿠키를 쓸 수 없어요(${e.message}) → 일반 유튜브 검색(Topic 채널 우선)으로 대신 찾습니다`); + const web = new YtWeb(); + return { searchSongs: (q) => web.searchSongs(q), kind: 'youtube' }; + } +} diff --git a/apps/server/src/db/migrations/002_songs.sql b/apps/server/src/db/migrations/002_songs.sql new file mode 100644 index 0000000..312192b --- /dev/null +++ b/apps/server/src/db/migrations/002_songs.sql @@ -0,0 +1,138 @@ +-- Song catalog (docs/07, docs/12 Q7). Stage 3: sources, YouTube matching, admin editing. +CREATE TABLE songs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + melon_id INTEGER UNIQUE, -- norabose ids are Melon song ids + title TEXT NOT NULL, + artist TEXT NOT NULL, -- display: main artists joined with ", " + year INTEGER, + release_date TEXT, + duration_sec INTEGER, + youtube_id TEXT, -- the video the quiz plays + youtube_status TEXT NOT NULL DEFAULT 'none' CHECK (youtube_status IN ('matched','candidate','none','dead')), + youtube_source TEXT, -- ytmusic | norabose | admin + youtube_channel TEXT, + youtube_title TEXT, + embeddable INTEGER, -- 1 ok, 0 embedding disabled, NULL unchecked + youtube_checked_at INTEGER, + mv_youtube_id TEXT, + highlight_sec REAL, + highlight_source TEXT, -- admin | heatmap | analysis | rule + peak_rank INTEGER, + hidden INTEGER NOT NULL DEFAULT 0, + note TEXT, + match_key TEXT NOT NULL, -- normalized title|artist for de-duplication + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + deleted_at INTEGER +); +CREATE INDEX songs_year ON songs(year); +CREATE INDEX songs_yt ON songs(youtube_status); +CREATE INDEX songs_match ON songs(match_key); + +CREATE TABLE artists ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + melon_artist_id INTEGER UNIQUE, + name TEXT NOT NULL, + kind TEXT, -- solo | group | unit + gender TEXT +); +CREATE INDEX artists_name ON artists(name); + +CREATE TABLE song_artists ( + song_id INTEGER NOT NULL REFERENCES songs(id) ON DELETE CASCADE, + artist_id INTEGER NOT NULL REFERENCES artists(id) ON DELETE CASCADE, + role TEXT NOT NULL DEFAULT 'main', -- main | feat + PRIMARY KEY (song_id, artist_id) +); +CREATE INDEX song_artists_artist ON song_artists(artist_id); + +CREATE TABLE artist_aliases ( + artist_id INTEGER NOT NULL REFERENCES artists(id) ON DELETE CASCADE, + alias TEXT NOT NULL, + PRIMARY KEY (artist_id, alias) +); + +CREATE TABLE song_aliases ( + song_id INTEGER NOT NULL REFERENCES songs(id) ON DELETE CASCADE, + part TEXT NOT NULL CHECK (part IN ('title','artist')), + alias TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'admin', + PRIMARY KEY (song_id, part, alias) +); + +CREATE TABLE genres ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + grp TEXT NOT NULL CHECK (grp IN ('region','style','theme')), + sort INTEGER NOT NULL, + hidden INTEGER NOT NULL DEFAULT 0 +); +INSERT INTO genres (id, name, grp, sort) VALUES + ('kpop','K-POP','region',10), ('jpop','J-POP','region',20), ('pop','POP','region',30), + ('ballad','발라드','style',110), ('dance','댄스','style',120), ('hiphop','랩/힙합','style',130), ('rnb','R&B/소울','style',140), + ('rock','록/메탈','style',150), ('indie','인디','style',160), ('folk','포크/블루스','style',170), ('electronic','일렉트로닉','style',180), + ('trot','성인가요/트로트','style',190), ('jazz','재즈','style',200), ('classical','클래식/뉴에이지','style',210), + ('ost','영화/드라마','theme',310), ('anime','애니/게임','theme',320), ('kids','동요/만화','theme',330), + ('girlgroup','걸그룹','theme',340), ('boygroup','보이그룹','theme',350); + +CREATE TABLE song_genres ( + song_id INTEGER NOT NULL REFERENCES songs(id) ON DELETE CASCADE, + genre_id TEXT NOT NULL REFERENCES genres(id) ON DELETE CASCADE, + source TEXT NOT NULL, + PRIMARY KEY (song_id, genre_id) +); +CREATE INDEX song_genres_genre ON song_genres(genre_id); + +-- A song may have several levels: one per scope (docs/12 Q7). scope = 'all' | 'genre:' | 'decade:<1990>'. +CREATE TABLE song_levels ( + song_id INTEGER NOT NULL REFERENCES songs(id) ON DELETE CASCADE, + scope TEXT NOT NULL, + level INTEGER NOT NULL CHECK (level BETWEEN 1 AND 10), + source TEXT NOT NULL, -- auto | admin + PRIMARY KEY (song_id, scope) +); +CREATE INDEX song_levels_level ON song_levels(level, scope); + +CREATE TABLE lyrics ( + song_id INTEGER PRIMARY KEY REFERENCES songs(id) ON DELETE CASCADE, + text TEXT NOT NULL, + source TEXT NOT NULL, + fetched_at INTEGER NOT NULL +); + +CREATE TABLE song_sources ( + song_id INTEGER NOT NULL REFERENCES songs(id) ON DELETE CASCADE, + source TEXT NOT NULL, -- sta | norabose | ... + ref TEXT NOT NULL, + PRIMARY KEY (song_id, source, ref) +); + +CREATE TABLE signals ( + song_id INTEGER NOT NULL REFERENCES songs(id) ON DELETE CASCADE, + kind TEXT NOT NULL, -- norabose_level_mask, norabose_peak, ... + value REAL NOT NULL, + fetched_at INTEGER NOT NULL, + PRIMARY KEY (song_id, kind) +); + +CREATE TABLE youtube_candidates ( + song_id INTEGER NOT NULL REFERENCES songs(id) ON DELETE CASCADE, + video_id TEXT NOT NULL, + title TEXT, + channel TEXT, + duration_sec INTEGER, + score REAL NOT NULL, + source TEXT NOT NULL, + PRIMARY KEY (song_id, video_id) +); + +CREATE TABLE collect_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + kind TEXT NOT NULL, + status TEXT NOT NULL, -- running | done | failed | stopped + done INTEGER NOT NULL DEFAULT 0, + total INTEGER NOT NULL DEFAULT 0, + log TEXT NOT NULL DEFAULT '', + started_at INTEGER NOT NULL, + finished_at INTEGER +); diff --git a/apps/server/src/http/admin.ts b/apps/server/src/http/admin.ts index 7de5d6d..c8ec436 100644 --- a/apps/server/src/http/admin.ts +++ b/apps/server/src/http/admin.ts @@ -6,6 +6,9 @@ import type { Audit, Roles } from '../admin/roles'; import { SiteSettingsSchema, type SettingsStore } from '../admin/settings'; import type { Sessions } from '../auth/sessions'; import { toPublicUser, type Users } from '../auth/users'; +import type { Database } from 'bun:sqlite'; +import { catalogStats, genreList, getSong, listSongs, updateSong } from '../songs/store'; +import { checkEmbed, type EmbedState } from '../collector/oembed'; export interface AdminDeps { users: Users; @@ -15,11 +18,43 @@ export interface AdminDeps { settings: SettingsStore; disconnectUser: (userId: string) => void; stats: () => Record; + db: Database; + /** Injected for tests. */ + checkEmbed?: (videoId: string) => Promise; } type Env = { Variables: { userId: string | null; role: Role } }; const DISCORD_ID = /^\d{17,20}$/; +const VIDEO_ID = /^[\w-]{11}$/; + +/** Accepts https://youtu.be/ID, youtube.com/watch?v=ID, music.youtube.com/watch?v=ID or the bare ID. */ +export function parseVideoId(s: string): string | null { + const t = s.trim(); + if (VIDEO_ID.test(t)) return t; + try { + const u = new URL(t); + const id = u.hostname === 'youtu.be' ? u.pathname.slice(1) : u.searchParams.get('v'); + return id && VIDEO_ID.test(id) ? id : null; + } catch { + return null; + } +} + +const SongPatchSchema = z.object({ + title: z.string().trim().min(1).max(200).optional(), + artist: z.string().trim().min(1).max(200).optional(), + year: z.number().int().min(1900).max(2100).nullable().optional(), + genres: z.array(z.string().max(30)).max(20).optional(), + level: z.number().int().min(1).max(10).nullable().optional(), + youtube: z.string().max(200).nullable().optional(), + highlightSec: z.number().min(0).max(3600).nullable().optional(), + hidden: z.boolean().optional(), + note: z.string().max(1000).nullable().optional(), + titleAliases: z.array(z.string().max(200)).max(30).optional(), + artistAliases: z.array(z.string().max(200)).max(30).optional(), + lyrics: z.string().max(20000).nullable().optional(), +}); export function createAdminApp(d: AdminDeps) { const app = new Hono(); @@ -53,6 +88,63 @@ export function createAdminApp(d: AdminDeps) { return c.json({ site: parsed.data }); }); + // ------------------------------------------------------------ songs (docs/02 노래 관리) + app.get('/catalog', (c) => c.json(catalogStats(d.db))); + app.get('/genres', (c) => c.json({ genres: genreList(d.db) })); + app.get('/songs', (c) => { + const q = (k: string) => c.req.query(k) || undefined; + const num = (k: string) => (q(k) ? Number(q(k)) || undefined : undefined); + const yt = q('youtube'); + const sort = q('sort'); + return c.json( + listSongs(d.db, { + q: q('q')?.trim().slice(0, 100), + genres: q('genres')?.split(',').filter(Boolean), + yearFrom: num('yearFrom'), + yearTo: num('yearTo'), + level: num('level'), + youtube: yt && ['matched', 'candidate', 'none', 'dead'].includes(yt) ? (yt as 'matched') : undefined, + source: q('source'), + hidden: q('hidden') === undefined ? undefined : q('hidden') === '1', + lyrics: q('lyrics') === undefined ? undefined : q('lyrics') === '1', + sort: sort && ['recent', 'year', 'level', 'peak', 'title'].includes(sort) ? (sort as 'recent') : undefined, + page: num('page'), + }), + ); + }); + app.get('/songs/:id', (c) => { + const r = getSong(d.db, Number(c.req.param('id'))); + return r ? c.json(r) : c.json({ error: '노래를 찾을 수 없어요.' }, 404); + }); + app.patch('/songs/:id', async (c) => { + const id = Number(c.req.param('id')); + const before = getSong(d.db, id); + if (!before) return c.json({ error: '노래를 찾을 수 없어요.' }, 404); + const parsed = SongPatchSchema.safeParse(await body(c)); + if (!parsed.success) return c.json({ error: `입력 값이 올바르지 않아요: ${parsed.error.issues.map((i) => i.path.join('.')).join(', ')}` }, 400); + const { youtube, ...rest } = parsed.data; + let youtubeId: string | null | undefined; + let embed: { author: string; title: string } | undefined; + if (youtube !== undefined) { + if (youtube === null || youtube.trim() === '') youtubeId = null; + else { + youtubeId = parseVideoId(youtube); + if (!youtubeId) return c.json({ error: '유튜브 주소를 알아볼 수 없어요. https://youtu.be/… 형식으로 넣어 주세요.' }, 400); + const e = await (d.checkEmbed ?? checkEmbed)(youtubeId); + if (e.state === 'noembed') return c.json({ error: '이 영상은 다른 사이트에서 재생할 수 없게 막혀 있어요(퍼가기 금지). 다른 영상을 골라 주세요.' }, 400); + if (e.state === 'dead') return c.json({ error: '없는 영상이에요.' }, 400); + if (e.state === 'error') return c.json({ error: '유튜브 확인에 실패했어요. 잠시 후 다시 시도해 주세요.' }, 502); + embed = { author: e.author, title: e.title }; + } + } + const genres = new Set(genreList(d.db).map((g) => g.id)); + if (rest.genres?.some((g) => !genres.has(g))) return c.json({ error: '없는 장르가 있어요.' }, 400); + updateSong(d.db, id, { ...rest, youtubeId }, embed); + const after = getSong(d.db, id)!; + d.audit.log(actor(c), 'song.update', String(id), { title: before.song.title, artist: before.song.artist }, parsed.data); + return c.json(after); + }); + // ------------------------------------------------------------ audit app.get('/audit', (c) => { const page = Math.max(0, Number(c.req.query('page') ?? 0) || 0); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index c4e4c9b..5905567 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -27,6 +27,8 @@ export interface StartOptions { log?: LogFn; staticDir?: string | null; discordExchange?: HttpDeps['discordExchange']; + /** YouTube embed check for admin edits (tests inject a fake). */ + checkEmbed?: import('./http/admin').AdminDeps['checkEmbed']; /** Upper bound for waiting on Bun's server.stop() (tests shorten it). */ stopWaitMs?: number; } @@ -70,6 +72,8 @@ export function startServer(opts: StartOptions) { removeGuest, admin: { audit, + db, + checkEmbed: opts.checkEmbed, disconnectUser: (id) => gateway.disconnectUser(id), stats: () => ({ connections: gateway.connectionCount, diff --git a/apps/server/src/songs.test.ts b/apps/server/src/songs.test.ts new file mode 100644 index 0000000..c790461 --- /dev/null +++ b/apps/server/src/songs.test.ts @@ -0,0 +1,56 @@ +import { afterEach, expect, test } from 'bun:test'; +import { Catalog } from './collector/catalog'; +import { parseVideoId } from './http/admin'; +import { TEST_SUPERADMINS, api, boot, discordLogin, guest } from './test-utils'; + +let s: ReturnType; +afterEach(async () => s && (await s.stop())); + +const fakeDiscord = async (code: string) => ({ id: code, username: 'u', global_name: '관리', avatar: null }); + +test('parseVideoId accepts the usual YouTube URL shapes', () => { + expect(parseVideoId('https://youtu.be/VO88htcm09E')).toBe('VO88htcm09E'); + expect(parseVideoId('https://music.youtube.com/watch?v=VO88htcm09E&si=x')).toBe('VO88htcm09E'); + expect(parseVideoId('VO88htcm09E')).toBe('VO88htcm09E'); + expect(parseVideoId('https://example.com/')).toBeNull(); +}); + +test('admin song list, filters and edit (YouTube embed checked)', async () => { + s = boot( + { + discordExchange: fakeDiscord, + checkEmbed: async (id) => (id === 'blockedvid1' ? { state: 'noembed' } : { state: 'ok', author: 'SG Wannabe - Topic', title: '광' }), + }, + { DISCORD_CLIENT_ID: 'cid', DISCORD_CLIENT_SECRET: 'sec' }, + ); + const c = new Catalog(s.db); + const [id] = c.upsert({ melonId: 1, title: '광 (狂)', artists: [{ name: 'SG워너비' }], year: 2005, genres: ['ballad', 'kpop'], source: 'norabose', ref: '1' }); + c.upsert({ melonId: 2, title: 'Love wins all', artists: [{ name: '아이유' }], year: 2024, genres: ['ballad', 'kpop'], source: 'norabose', ref: '2' }); + c.autoLevel(id, 'all', 3); + const g = await guest(s.base, '손님'); + expect((await api(s.base, '/api/admin/songs', { cookie: g.cookie })).res.status).toBe(403); + const sa = await discordLogin(s.base, TEST_SUPERADMINS[0]); + const all = await api(s.base, '/api/admin/songs', { cookie: sa.cookie }); + expect(all.body.total).toBe(2); + const q = await api(s.base, `/api/admin/songs?q=${encodeURIComponent('SG')}&genres=ballad&yearFrom=2000&yearTo=2010&level=3`, { cookie: sa.cookie }); + expect(q.body.rows.map((r: any) => r.id)).toEqual([id]); + expect((await api(s.base, '/api/admin/catalog', { cookie: sa.cookie })).body).toMatchObject({ total: 2, youtube: { none: 2 } }); + + const blocked = await api(s.base, `/api/admin/songs/${id}`, { method: 'PATCH', cookie: sa.cookie, body: JSON.stringify({ youtube: 'https://youtu.be/blockedvid1' }) }); + expect(blocked.res.status).toBe(400); + const ok = await api(s.base, `/api/admin/songs/${id}`, { + method: 'PATCH', + cookie: sa.cookie, + body: JSON.stringify({ youtube: 'https://youtu.be/VO88htcm09E', level: 2, titleAliases: ['광', 'Crazy'], highlightSec: 75.8, genres: ['ballad'] }), + }); + expect(ok.res.status).toBe(200); + expect(ok.body.song).toMatchObject({ youtube_id: 'VO88htcm09E', youtube_status: 'matched', youtube_source: 'admin', youtube_channel: 'SG Wannabe - Topic', highlight_sec: 75.8, highlight_source: 'admin' }); + expect(ok.body.levels).toEqual([{ scope: 'all', level: 2, source: 'admin' }]); + expect(ok.body.aliases.map((a: any) => a.alias).sort()).toEqual(['Crazy', '광']); + expect(ok.body.genres.map((g: any) => g.genre_id)).toEqual(['ballad']); + expect((await api(s.base, `/api/admin/songs?q=Crazy`, { cookie: sa.cookie })).body.total).toBe(1); + const bad = await api(s.base, `/api/admin/songs/${id}`, { method: 'PATCH', cookie: sa.cookie, body: JSON.stringify({ genres: ['nope'] }) }); + expect(bad.res.status).toBe(400); + const audit = await api(s.base, '/api/admin/audit', { cookie: sa.cookie }); + expect(audit.body.entries[0]).toMatchObject({ action: 'song.update', target: String(id) }); +}); diff --git a/apps/server/src/songs/store.ts b/apps/server/src/songs/store.ts new file mode 100644 index 0000000..0ea2e44 --- /dev/null +++ b/apps/server/src/songs/store.ts @@ -0,0 +1,190 @@ +/** Song catalog queries for the admin pages (docs/02 노래 관리 화면). */ +import type { Database } from 'bun:sqlite'; +import { songMatchKey } from '@mq/shared'; + +export function catalogStats(db: Database) { + const one = (sql: string) => db.query(sql).get()!; + const songs = one<{ total: number; matched: number; candidate: number; none: number; dead: number; hidden: number }>( + `SELECT COUNT(*) total, SUM(youtube_status='matched') matched, SUM(youtube_status='candidate') candidate, + SUM(youtube_status='none') none, SUM(youtube_status='dead') dead, SUM(hidden) hidden FROM songs WHERE deleted_at IS NULL`, + ); + const lyrics = one<{ n: number }>('SELECT COUNT(*) n FROM lyrics l JOIN songs s ON s.id = l.song_id WHERE s.deleted_at IS NULL').n; + const sources = db.query<{ source: string; n: number }, []>('SELECT source, COUNT(DISTINCT song_id) n FROM song_sources GROUP BY source').all(); + const tried = one<{ n: number }>("SELECT COUNT(DISTINCT song_id) n FROM signals WHERE kind LIKE 'yt_tried_%'").n; + const jobs = db.query<{ id: number; kind: string; status: string; done: number; total: number; started_at: number; finished_at: number | null }, []>( + 'SELECT id, kind, status, done, total, started_at, finished_at FROM collect_jobs ORDER BY id DESC LIMIT 5', + ).all(); + const z = (n: number | null) => n ?? 0; + return { + total: songs.total, + youtube: { matched: z(songs.matched), candidate: z(songs.candidate), none: z(songs.none), dead: z(songs.dead), tried }, + hidden: z(songs.hidden), + lyrics, + sources: Object.fromEntries(sources.map((r) => [r.source, r.n])), + jobs, + }; +} + +export interface SongFilter { + q?: string; + genres?: string[]; + yearFrom?: number; + yearTo?: number; + level?: number; + youtube?: 'matched' | 'candidate' | 'none' | 'dead'; + source?: string; + hidden?: boolean; + lyrics?: boolean; + sort?: 'recent' | 'year' | 'level' | 'peak' | 'title'; + page?: number; +} + +export const PAGE = 50; + +export function listSongs(db: Database, f: SongFilter) { + const where: string[] = ['s.deleted_at IS NULL']; + const args: (string | number)[] = []; + if (f.q) { + const like = `%${f.q.replace(/[%_\\]/g, (m) => '\\' + m)}%`; + where.push(`(s.title LIKE ? ESCAPE '\\' OR s.artist LIKE ? ESCAPE '\\' OR EXISTS (SELECT 1 FROM song_aliases a WHERE a.song_id = s.id AND a.alias LIKE ? ESCAPE '\\') OR s.youtube_id = ? OR CAST(s.melon_id AS TEXT) = ?)`); + args.push(like, like, like, f.q, f.q); + } + if (f.genres?.length) { + where.push(`EXISTS (SELECT 1 FROM song_genres g WHERE g.song_id = s.id AND g.genre_id IN (${f.genres.map(() => '?').join(',')}))`); + args.push(...f.genres); + } + if (f.yearFrom) (where.push('s.year >= ?'), args.push(f.yearFrom)); + if (f.yearTo) (where.push('s.year <= ?'), args.push(f.yearTo)); + if (f.level) (where.push('EXISTS (SELECT 1 FROM song_levels l WHERE l.song_id = s.id AND l.level = ?)'), args.push(f.level)); + if (f.youtube) (where.push('s.youtube_status = ?'), args.push(f.youtube)); + if (f.source) (where.push('EXISTS (SELECT 1 FROM song_sources x WHERE x.song_id = s.id AND x.source = ?)'), args.push(f.source)); + if (f.hidden !== undefined) (where.push('s.hidden = ?'), args.push(f.hidden ? 1 : 0)); + if (f.lyrics !== undefined) where.push(`${f.lyrics ? '' : 'NOT '}EXISTS (SELECT 1 FROM lyrics y WHERE y.song_id = s.id)`); + const order = { + recent: 's.id DESC', + year: 's.year DESC NULLS LAST, s.id', + level: 'lv.level NULLS LAST, s.id', + peak: 's.peak_rank NULLS LAST, s.id', + title: 's.title COLLATE NOCASE', + }[f.sort ?? 'recent']; + const w = where.join(' AND '); + const total = db.query<{ n: number }, (string | number)[]>(`SELECT COUNT(*) n FROM songs s WHERE ${w}`).get(...args)!.n; + const page = Math.max(0, f.page ?? 0); + const rows = db + .query, (string | number)[]>( + `SELECT s.id, s.melon_id, s.title, s.artist, s.year, s.duration_sec, s.youtube_id, s.youtube_status, s.youtube_source, s.youtube_channel, + s.peak_rank, s.hidden, s.highlight_sec, lv.level, + (SELECT group_concat(genre_id) FROM song_genres g WHERE g.song_id = s.id) genres, + (SELECT group_concat(DISTINCT source) FROM song_sources x WHERE x.song_id = s.id) sources, + EXISTS (SELECT 1 FROM lyrics y WHERE y.song_id = s.id) has_lyrics + FROM songs s LEFT JOIN song_levels lv ON lv.song_id = s.id AND lv.scope = 'all' + WHERE ${w} ORDER BY ${order} LIMIT ${PAGE} OFFSET ${page * PAGE}`, + ) + .all(...args) + .map((r) => ({ + ...r, + genres: r.genres ? String(r.genres).split(',') : [], + sources: r.sources ? String(r.sources).split(',') : [], + hidden: !!r.hidden, + has_lyrics: !!r.has_lyrics, + })); + return { total, page, pageSize: PAGE, rows }; +} + +export function getSong(db: Database, id: number) { + const song = db.query, [number]>('SELECT * FROM songs WHERE id = ? AND deleted_at IS NULL').get(id); + if (!song) return null; + return { + song: { ...(song as { id: number; title: string; artist: string } & Record), hidden: !!song.hidden }, + genres: db.query<{ genre_id: string; source: string }, [number]>('SELECT genre_id, source FROM song_genres WHERE song_id = ?').all(id), + levels: db.query<{ scope: string; level: number; source: string }, [number]>('SELECT scope, level, source FROM song_levels WHERE song_id = ? ORDER BY scope').all(id), + aliases: db.query<{ part: string; alias: string; source: string }, [number]>('SELECT part, alias, source FROM song_aliases WHERE song_id = ?').all(id), + lyrics: db.query<{ text: string; source: string }, [number]>('SELECT text, source FROM lyrics WHERE song_id = ?').get(id), + sources: db.query<{ source: string; ref: string }, [number]>('SELECT source, ref FROM song_sources WHERE song_id = ?').all(id), + candidates: db + .query, [number]>('SELECT video_id, title, channel, duration_sec, score, source FROM youtube_candidates WHERE song_id = ? ORDER BY score DESC') + .all(id), + signals: db.query<{ kind: string; value: number }, [number]>('SELECT kind, value FROM signals WHERE song_id = ?').all(id), + }; +} + +export interface SongPatch { + title?: string; + artist?: string; + year?: number | null; + genres?: string[]; + level?: number | null; + youtubeId?: string | null; + highlightSec?: number | null; + hidden?: boolean; + note?: string | null; + titleAliases?: string[]; + artistAliases?: string[]; + lyrics?: string | null; +} + +/** Admin edit. Anything set here is marked as admin so collectors never overwrite it. */ +export function updateSong(db: Database, id: number, p: SongPatch, embed?: { author: string; title: string }, now = Date.now()) { + db.transaction(() => { + const cur = db.query<{ title: string; artist: string }, [number]>('SELECT title, artist FROM songs WHERE id = ?').get(id)!; + const title = p.title ?? cur.title; + const artist = p.artist ?? cur.artist; + db.query( + `UPDATE songs SET title = ?, artist = ?, match_key = ?, year = CASE WHEN ? THEN ? ELSE year END, + highlight_sec = CASE WHEN ? THEN ? ELSE highlight_sec END, highlight_source = CASE WHEN ? THEN 'admin' ELSE highlight_source END, + hidden = COALESCE(?, hidden), note = CASE WHEN ? THEN ? ELSE note END, updated_at = ? WHERE id = ?`, + ).run( + title, + artist, + songMatchKey(title, artist), + p.year !== undefined ? 1 : 0, + p.year ?? null, + p.highlightSec !== undefined ? 1 : 0, + p.highlightSec ?? null, + p.highlightSec !== undefined && p.highlightSec !== null ? 1 : 0, + p.hidden === undefined ? null : p.hidden ? 1 : 0, + p.note !== undefined ? 1 : 0, + p.note ?? null, + now, + id, + ); + if (p.youtubeId !== undefined) { + if (p.youtubeId === null) db.query("UPDATE songs SET youtube_id = NULL, youtube_status = 'none', youtube_source = 'admin' WHERE id = ?").run(id); + else + db.query( + "UPDATE songs SET youtube_id = ?, youtube_status = 'matched', youtube_source = 'admin', youtube_channel = ?, youtube_title = ?, embeddable = 1, youtube_checked_at = ? WHERE id = ?", + ).run(p.youtubeId, embed?.author ?? null, embed?.title ?? null, now, id); + } + if (p.genres) { + db.query('DELETE FROM song_genres WHERE song_id = ?').run(id); + for (const g of p.genres) db.query("INSERT INTO song_genres (song_id, genre_id, source) VALUES (?, ?, 'admin')").run(id, g); + } + if (p.level !== undefined) { + if (p.level === null) db.query("DELETE FROM song_levels WHERE song_id = ? AND scope = 'all'").run(id); + else + db.query("INSERT INTO song_levels (song_id, scope, level, source) VALUES (?, 'all', ?, 'admin') ON CONFLICT(song_id, scope) DO UPDATE SET level = excluded.level, source = 'admin'").run(id, p.level); + } + for (const [part, list] of [ + ['title', p.titleAliases], + ['artist', p.artistAliases], + ] as const) { + if (!list) continue; + db.query('DELETE FROM song_aliases WHERE song_id = ? AND part = ?').run(id, part); + for (const a of new Set(list.map((x) => x.trim()).filter(Boolean))) db.query("INSERT INTO song_aliases (song_id, part, alias, source) VALUES (?, ?, ?, 'admin')").run(id, part, a); + } + if (p.lyrics !== undefined) { + if (p.lyrics) db.query("INSERT INTO lyrics (song_id, text, source, fetched_at) VALUES (?, ?, 'admin', ?) ON CONFLICT(song_id) DO UPDATE SET text = excluded.text, source = 'admin', fetched_at = excluded.fetched_at").run(id, p.lyrics, now); + else db.query('DELETE FROM lyrics WHERE song_id = ?').run(id); + } + })(); +} + +export function genreList(db: Database) { + return db + .query<{ id: string; name: string; grp: string; sort: number; hidden: number; songs: number }, []>( + `SELECT g.*, (SELECT COUNT(*) FROM song_genres sg JOIN songs s ON s.id = sg.song_id WHERE sg.genre_id = g.id AND s.deleted_at IS NULL) songs + FROM genres g ORDER BY sort`, + ) + .all() + .map((g) => ({ ...g, hidden: !!g.hidden })); +} diff --git a/apps/web/src/pages/admin/AdminApp.tsx b/apps/web/src/pages/admin/AdminApp.tsx index fff9fb4..9e532b7 100644 --- a/apps/web/src/pages/admin/AdminApp.tsx +++ b/apps/web/src/pages/admin/AdminApp.tsx @@ -9,6 +9,7 @@ import { ApiError, apiCall, discordLoginUrl } from '../../net/api'; import { useSession } from '../../store/session'; import { toast } from '../../store/toast'; import { ConfirmSheet } from '../../components/ui'; +import { CollectPage, SongsPage } from './SongsPage'; const fmt = (ms: number | null | undefined) => (ms ? new Date(ms).toLocaleString('ko-KR', { dateStyle: 'short', timeStyle: 'short' }) : '-'); @@ -25,7 +26,7 @@ function useLoad(path: string | null) { return { data, error, reload, setData }; } -const SOON = ['노래', '가수', '장르', '수집', '통계', '방']; +const SOON = ['가수', '장르', '통계', '방']; export default function AdminApp() { const me = useSession((s) => s.me); @@ -60,6 +61,8 @@ export default function AdminApp() { 📊 대시보드
노래 DB
+ 🎵 노래 + 📥 수집 {SOON.map((s) => ( {s} 준비 중 @@ -74,6 +77,8 @@ export default function AdminApp() {
} /> + } /> + } /> } /> {isSuper && } />} {isSuper && } />} @@ -89,6 +94,7 @@ function Dashboard({ role }: { role: Role }) { const { data, error } = useLoad<{ users: { total: number; guests: number; members: number; banned: number }; connections: number; uptimeSec: number; memoryMb: number }>( '/api/admin/overview', ); + const { data: cat } = useLoad<{ total: number; youtube: { matched: number }; lyrics: number }>('/api/admin/catalog'); return ( <>

대시보드

@@ -124,10 +130,23 @@ function Dashboard({ role }: { role: Role }) { )} -
-

다음에 생기는 것

-

노래 수, 유튜브 매칭률, 가사 보유율, 진행 중인 방, 오늘 플레이 수는 노래 DB와 방 기능이 들어오면 여기에 보여요.

-
+ {cat && ( +
+
+ {cat.total.toLocaleString()} + 전체 곡 +
+
+ {cat.total ? Math.round((cat.youtube.matched / cat.total) * 100) : 0}% + 유튜브 확정률 +
+
+ {cat.total ? Math.round((cat.lyrics / cat.total) * 100) : 0}% + 가사 보유율 +
+
+ )} +

진행 중인 방과 오늘 플레이 수는 방 기능이 들어오면 여기에 보여요.

); } diff --git a/apps/web/src/pages/admin/SongsPage.tsx b/apps/web/src/pages/admin/SongsPage.tsx new file mode 100644 index 0000000..3d806a9 --- /dev/null +++ b/apps/web/src/pages/admin/SongsPage.tsx @@ -0,0 +1,492 @@ +/** + * 관리자 → 노래 (docs/02 노래 관리 화면): 검색·필터·정렬 표 + 오른쪽 편집 패널(유튜브 미리듣기·후보 고르기·하이라이트·별칭). + */ +import { useCallback, useEffect, useMemo, useState, type FormEvent } from 'react'; +import { apiCall } from '../../net/api'; +import { toast } from '../../store/toast'; + +interface Genre { + id: string; + name: string; + grp: 'region' | 'style' | 'theme'; + songs: number; +} +interface Row { + id: number; + melon_id: number | null; + title: string; + artist: string; + year: number | null; + duration_sec: number | null; + youtube_id: string | null; + youtube_status: 'matched' | 'candidate' | 'none' | 'dead'; + youtube_source: string | null; + youtube_channel: string | null; + peak_rank: number | null; + level: number | null; + genres: string[]; + sources: string[]; + has_lyrics: boolean; + hidden: boolean; +} +interface Detail { + song: Row & { youtube_title: string | null; highlight_sec: number | null; highlight_source: string | null; note: string | null; mv_youtube_id: string | null }; + genres: { genre_id: string }[]; + levels: { scope: string; level: number; source: string }[]; + aliases: { part: 'title' | 'artist'; alias: string }[]; + lyrics: { text: string } | null; + sources: { source: string; ref: string }[]; + candidates: { video_id: string; title: string | null; channel: string | null; duration_sec: number | null; score: number; source: string }[]; +} +interface Stats { + total: number; + youtube: { matched: number; candidate: number; none: number; dead: number; tried: number }; + lyrics: number; + sources: Record; + jobs: { id: number; kind: string; status: string; done: number; total: number; started_at: number; finished_at: number | null }[]; +} + +const YT_LABEL: Record = { + matched: ['확정', 'ok'], + candidate: ['후보', 'warn'], + none: ['없음', ''], + dead: ['재생 불가', 'warn'], +}; +const SOURCE_LABEL: Record = { norabose: '노라보세', sta: '스타맵' }; +const mmss = (s: number | null) => (s == null ? '-' : `${Math.floor(s / 60)}:${String(Math.round(s % 60)).padStart(2, '0')}`); +const pct = (n: number, d: number) => (d ? `${Math.round((n / d) * 100)}%` : '-'); + +export function SongsPage() { + const [genres, setGenres] = useState([]); + const [stats, setStats] = useState(null); + const [q, setQ] = useState(''); + const [f, setF] = useState({ q: '', genres: [] as string[], yearFrom: '', yearTo: '', level: '', youtube: '', source: '', sort: 'recent', page: 0 }); + const [data, setData] = useState<{ total: number; rows: Row[]; pageSize: number } | null>(null); + const [selected, setSelected] = useState(null); + + const loadStats = useCallback(() => void apiCall('GET', '/api/admin/catalog').then(setStats).catch(() => {}), []); + useEffect(() => { + void apiCall<{ genres: Genre[] }>('GET', '/api/admin/genres').then((r) => setGenres(r.genres)); + loadStats(); + }, [loadStats]); + + const query = useMemo(() => { + const p = new URLSearchParams(); + if (f.q) p.set('q', f.q); + if (f.genres.length) p.set('genres', f.genres.join(',')); + for (const k of ['yearFrom', 'yearTo', 'level', 'youtube', 'source', 'sort'] as const) if (f[k]) p.set(k, f[k]); + p.set('page', String(f.page)); + return p.toString(); + }, [f]); + const reload = useCallback(() => { + apiCall<{ total: number; rows: Row[]; pageSize: number }>('GET', `/api/admin/songs?${query}`) + .then(setData) + .catch((e: Error) => toast(e.message, 'error')); + }, [query]); + useEffect(reload, [reload]); + + const set = (patch: Partial) => setF((x) => ({ ...x, page: 0, ...patch })); + const toggleGenre = (id: string) => set({ genres: f.genres.includes(id) ? f.genres.filter((g) => g !== id) : [...f.genres, id] }); + const pages = data ? Math.max(1, Math.ceil(data.total / data.pageSize)) : 1; + + return ( +
+
+

노래

+ {stats && ( +
+
+ {stats.total.toLocaleString()} + 전체 곡 +
+
+ {pct(stats.youtube.matched, stats.total)} + 유튜브 확정 {stats.youtube.matched.toLocaleString()} +
+
+ {stats.youtube.candidate.toLocaleString()} + 후보 확인 대기 +
+
+ {stats.youtube.none.toLocaleString()} + 유튜브 없음 +
+
+ {pct(stats.lyrics, stats.total)} + 가사 보유 {stats.lyrics.toLocaleString()} +
+
+ )} +
{ + e.preventDefault(); + set({ q: q.trim() }); + }} + > + setQ(e.target.value)} aria-label="노래 검색" /> + + {data && {data.total.toLocaleString()}곡} +
+
+ {genres.map((g) => ( + + ))} +
+
+ + + + + +
+
+ + + + + + + + + + + + + + + {data?.rows.map((r) => ( + setSelected(r.id)} style={{ cursor: 'pointer' }}> + + + + + + + + + + ))} + +
제목가수연도장르난이도유튜브가사출처
+ + {r.hidden && 숨김} + {r.artist}{r.year ?? '-'}{r.genres.map((g) => genres.find((x) => x.id === g)?.name ?? g).join(', ')}{r.level ? `★${r.level}` : '-'} + {YT_LABEL[r.youtube_status][0]} + {r.has_lyrics ? '있음' : '-'}{r.sources.map((s) => SOURCE_LABEL[s] ?? s).join(', ')}
+
+
+ + + {f.page + 1} / {pages} + + +
+
+ {selected && ( + setSelected(null)} + onSaved={() => { + reload(); + loadStats(); + }} + /> + )} +
+ ); +} + +function SongEditor({ id, genres, onClose, onSaved }: { id: number; genres: Genre[]; onClose: () => void; onSaved: () => void }) { + const [d, setD] = useState(null); + const [form, setForm] = useState({ title: '', artist: '', year: '', level: '', youtube: '', highlight: '', titleAliases: '', artistAliases: '', note: '', hidden: false, genres: [] as string[] }); + const [busy, setBusy] = useState(false); + const load = useCallback(() => { + apiCall('GET', `/api/admin/songs/${id}`) + .then((x) => { + setD(x); + const lv = x.levels.find((l) => l.scope === 'all'); + setForm({ + title: x.song.title, + artist: x.song.artist, + year: x.song.year ? String(x.song.year) : '', + level: lv ? String(lv.level) : '', + youtube: x.song.youtube_id ? `https://youtu.be/${x.song.youtube_id}` : '', + highlight: x.song.highlight_sec != null ? String(x.song.highlight_sec) : '', + titleAliases: x.aliases.filter((a) => a.part === 'title').map((a) => a.alias).join(', '), + artistAliases: x.aliases.filter((a) => a.part === 'artist').map((a) => a.alias).join(', '), + note: x.song.note ?? '', + hidden: x.song.hidden, + genres: x.genres.map((g) => g.genre_id), + }); + }) + .catch((e: Error) => toast(e.message, 'error')); + }, [id]); + useEffect(load, [load]); + if (!d) return ; + const split = (s: string) => s.split(',').map((x) => x.trim()).filter(Boolean); + const save = async (e: FormEvent) => { + e.preventDefault(); + setBusy(true); + try { + const curYt = d.song.youtube_id ? `https://youtu.be/${d.song.youtube_id}` : ''; + await apiCall('PATCH', `/api/admin/songs/${id}`, { + title: form.title, + artist: form.artist, + year: form.year ? Number(form.year) : null, + level: form.level ? Number(form.level) : null, + genres: form.genres, + ...(form.youtube.trim() !== curYt || d.song.youtube_status !== 'matched' ? { youtube: form.youtube.trim() || null } : {}), + highlightSec: form.highlight ? Number(form.highlight) : null, + titleAliases: split(form.titleAliases), + artistAliases: split(form.artistAliases), + note: form.note || null, + hidden: form.hidden, + }); + toast('저장했어요.'); + load(); + onSaved(); + } catch (ex) { + toast((ex as Error).message, 'error'); + } finally { + setBusy(false); + } + }; + const videoId = /youtu\.be\/([\w-]{11})/.exec(form.youtube)?.[1] ?? d.song.youtube_id; + return ( +