diff --git a/apps/server/src/collector/catalog.ts b/apps/server/src/collector/catalog.ts index 6e9d464..3c7a947 100644 --- a/apps/server/src/collector/catalog.ts +++ b/apps/server/src/collector/catalog.ts @@ -1,6 +1,7 @@ /** Writes into the song catalog (docs/07). Idempotent: re-running an import never duplicates songs. */ import type { Database } from 'bun:sqlite'; import { songMatchKey } from '@mq/shared'; +import { decodeEntities } from './text'; export interface SongInput { melonId?: number | null; @@ -31,7 +32,9 @@ export class Catalog { } /** Finds the song by Melon id, else by match key; creates it when new. Returns [id, created]. */ - upsert(s: SongInput, now = Date.now()): [number, boolean] { + upsert(input: SongInput, now = Date.now()): [number, boolean] { + // Sources hand us HTML-escaped text now and then (norabose `Salty & Sweet`): store what people read. + const s = { ...input, title: decodeEntities(input.title), artists: input.artists.map((a) => ({ ...a, name: decodeEntities(a.name) })) }; return this.db.transaction((): [number, boolean] => { const artist = s.artists.map((a) => a.name).join(', '); const key = songMatchKey(s.title, artist); diff --git a/apps/server/src/collector/charts.ts b/apps/server/src/collector/charts.ts index 7e0606a..dcffb48 100644 --- a/apps/server/src/collector/charts.ts +++ b/apps/server/src/collector/charts.ts @@ -6,6 +6,7 @@ import type { Database } from 'bun:sqlite'; import { songMatchKey } from '@mq/shared'; import { Catalog, Job, sleep } from './catalog'; +import { decodeEntities } from './text'; export const BROWSER_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0 Safari/537.36'; @@ -18,15 +19,7 @@ export interface ChartRow { } const decode = (s: string) => - s - .replace(/<[^>]+>/g, '') - .replace(/ /g, ' ') - .replace(/&/g, '&') - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/"/g, '"') - .replace(/'|'/g, "'") - .replace(/&#(\d+);/g, (_, n: string) => String.fromCodePoint(Number(n))) + decodeEntities(s.replace(/<[^>]+>/g, '')) .replace(/\s+/g, ' ') .trim(); diff --git a/apps/server/src/collector/cli.ts b/apps/server/src/collector/cli.ts index dc21493..a2bc52e 100644 --- a/apps/server/src/collector/cli.ts +++ b/apps/server/src/collector/cli.ts @@ -7,6 +7,7 @@ * bun apps/server/src/collector/cli.ts melon-detail [--limit N] * bun apps/server/src/collector/cli.ts heatmap [--limit N] * bun apps/server/src/collector/cli.ts levels + * bun apps/server/src/collector/cli.ts fix-text (one-off: decode `'`/`&` saved in titles, let 원곡 찾기 retry them) * bun apps/server/src/collector/cli.ts stats * In the container: docker exec music_quiz-app-1 bun apps/server/src/collector/cli.ts … */ @@ -21,6 +22,7 @@ import { currentJob } from './catalog'; import { collectCharts, fillMelonDetails } from './charts'; import { collectHeatmaps } from './heatmap'; import { computeLevels } from './levels'; +import { fixEntities } from './text'; const [cmd, ...rest] = process.argv.slice(2); const flag = (name: string) => { @@ -65,11 +67,14 @@ switch (cmd) { case 'levels': console.log(computeLevels(db)); break; + case 'fix-text': + console.log(JSON.stringify(fixEntities(db), null, 1)); + break; case 'stats': console.log(JSON.stringify(catalogStats(db), null, 1)); break; default: - console.log('usage: cli.ts import-norabose|import-sta|match-youtube|charts|melon-detail|heatmap|levels|stats'); + console.log('usage: cli.ts import-norabose|import-sta|match-youtube|charts|melon-detail|heatmap|levels|fix-text|stats'); process.exitCode = 1; } db.close(); diff --git a/apps/server/src/collector/text.test.ts b/apps/server/src/collector/text.test.ts new file mode 100644 index 0000000..ee85855 --- /dev/null +++ b/apps/server/src/collector/text.test.ts @@ -0,0 +1,66 @@ +/** `'` in song titles (2026-10-19): decoding on the way in and repairing what was already saved. */ +import { expect, test } from 'bun:test'; +import { join } from 'node:path'; +import { songMatchKey } from '@mq/shared'; +import { openDb } from '../db'; +import { tempDbPath } from '../test-utils'; +import { Catalog } from './catalog'; +import { parseMelon, saveChart } from './charts'; +import { decodeEntities, fixEntities } from './text'; + +test('entities: hex, decimal and named, one pass, unknown ones kept', () => { + expect(decodeEntities('Somebody's Watching Me')).toBe("Somebody's Watching Me"); + expect(decodeEntities('AC/DC 'n' Salty & Sweet "x" <3>')).toBe(`AC/DC 'n' Salty & Sweet "x" <3>`); + expect(decodeEntities('&#x27;')).toBe('''); + expect(decodeEntities('R&B & Soul; &bogus; �')).toBe('R&B & Soul; &bogus; �'); +}); + +test('Melon chart rows with hex entities are stored readable', async () => { + // the saved Melon page with the first title the way Melon escapes an apostrophe + const html = (await Bun.file(join(import.meta.dir, 'fixtures', 'melon-chart.html')).text()).replace(/" >죄와벌 Somebody's Watching Me <'); + expect(parseMelon(html)[0]!.title).toBe("Somebody's Watching Me"); + // whatever the page shape, the catalog itself never stores escaped text + const db = openDb(tempDbPath()); + const [id] = new Catalog(db).upsert({ melonId: 1, title: 'Salty & Sweet', artists: [{ name: 'Tom 'n' Jerry' }], source: 'norabose', ref: '1' }); + expect(db.query<{ title: string; artist: string; match_key: string }, [number]>('SELECT title, artist, match_key FROM songs WHERE id = ?').get(id)).toEqual({ + title: 'Salty & Sweet', + artist: "Tom 'n' Jerry", + match_key: songMatchKey('Salty & Sweet', "Tom 'n' Jerry"), + }); +}); + +test('fix-text repairs saved songs, chart rows and artists, retries 원곡 찾기, merges the duplicates it uncovers', () => { + const db = openDb(tempDbPath()); + const now = Date.now(); + const add = (title: string, artist: string, status: string, source: string | null = null) => + Number( + db + .query("INSERT INTO songs (title, artist, match_key, youtube_status, youtube_source, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 0, 0)") + .run(title, artist, songMatchKey(title, artist), status, source).lastInsertRowid, + ); + const broken = add('Somebody's Watching Me', 'Rockwell', 'none'); + const matched = add('I'm On Fire', 'Bruce Springsteen', 'matched', 'ytmusic'); + const admin = add('You & Me', '제니', 'candidate', 'admin'); + const original = add("I'm On Fire", 'Bruce Springsteen', 'matched', 'ytmusic'); + const clean = add('Hype Boy', 'NewJeans', 'none'); + for (const id of [broken, matched, admin, clean]) db.query("INSERT INTO signals (song_id, kind, value, fetched_at) VALUES (?, 'yt_tried_ytmusic', ?, ?)").run(id, now, now); + db.query("INSERT INTO artists (name) VALUES ('Tom & Jerry')").run(); + db.query("INSERT INTO chart_entries (source, chart, period, rank, raw_title, raw_artist, song_id, fetched_at) VALUES ('melon', 'year-pop', '1984', 1, 'I'm On Fire', 'Bruce Springsteen', ?, 0)").run(matched); + saveChart(db, { source: 'genie', chart: 'year', period: '1984', year: 1984 }, [{ rank: 1, ref: '1', title: 'Somebody's', artists: [{ name: 'A & B' }] }]); + + const r = fixEntities(db); + expect(r).toMatchObject({ songs: 3, retry: 1, artists: 1, chartRows: 2 }); + expect(r.duplicates).toEqual([{ id: matched, title: "I'm On Fire", artist: 'Bruce Springsteen', sameAs: [original] }]); + // the copy went to the trash and its chart rows follow the song that stays + expect(db.query<{ deleted: number }, [number]>('SELECT deleted_at IS NOT NULL deleted FROM songs WHERE id = ?').get(matched)!.deleted).toBe(1); + const row = (id: number) => db.query<{ title: string; match_key: string }, [number]>('SELECT title, match_key FROM songs WHERE id = ?').get(id)!; + expect(row(broken)).toEqual({ title: "Somebody's Watching Me", match_key: songMatchKey("Somebody's Watching Me", 'Rockwell') }); + expect(row(admin).title).toBe('You & Me'); + const tried = db.query<{ song_id: number }, []>("SELECT song_id FROM signals WHERE kind = 'yt_tried_ytmusic' ORDER BY song_id").all().map((x) => x.song_id); + expect(tried).toEqual([admin, original, clean]); // only the unsettled, non-admin song is retried (the copy's mark moved over) + expect(db.query<{ raw_title: string; raw_artist: string; song_id: number | null }, []>('SELECT raw_title, raw_artist, song_id FROM chart_entries ORDER BY source').all()).toEqual([ + { raw_title: "Somebody's", raw_artist: 'A & B', song_id: null }, + { raw_title: "I'm On Fire", raw_artist: 'Bruce Springsteen', song_id: original }, + ]); + expect(fixEntities(db)).toMatchObject({ songs: 0, retry: 0, artists: 0, chartRows: 0 }); +}); diff --git a/apps/server/src/collector/text.ts b/apps/server/src/collector/text.ts new file mode 100644 index 0000000..9755e73 --- /dev/null +++ b/apps/server/src/collector/text.ts @@ -0,0 +1,69 @@ +/** HTML entities in collected song text (`Somebody's`, `Salty & Sweet`): decoding and the one-off DB repair. */ +import type { Database } from 'bun:sqlite'; +import { songMatchKey } from '@mq/shared'; +import { mergeSongs } from '../songs/admin-ops'; + +const NAMED: Record = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' }; + +/** One pass, so `&#x27;` becomes the literal text `'` (not `'`). Unknown entities stay as they are. */ +export function decodeEntities(s: string): string { + return s.replace(/&(#[xX][0-9a-fA-F]+|#\d+|[a-zA-Z]+);/g, (m, e: string) => { + if (e[0] !== '#') return NAMED[e.toLowerCase()] ?? m; + const n = e[1] === 'x' || e[1] === 'X' ? parseInt(e.slice(2), 16) : Number(e.slice(1)); + return n > 0 && n <= 0x10ffff ? String.fromCodePoint(n) : m; + }); +} + +const HAS_ENTITY = /&(#[xX][0-9a-fA-F]+|#\d+|[a-zA-Z]+);/; + +/** + * Repairs songs/artists/chart rows saved with entities: decodes the text, recomputes the match key and lets + * 원곡 찾기 try the song again (its title never matched). A song whose fixed key now equals an older song's was a + * duplicate made by the broken key: it is merged into the oldest one (관리자 → 중복 합치기, the copy goes to the trash). + */ +export function fixEntities(db: Database, now = Date.now()) { + return db.transaction(() => { + const out = { songs: 0, retry: 0, artists: 0, chartRows: 0, duplicates: [] as { id: number; title: string; artist: string; sameAs: number[] }[] }; + const songs = db + .query<{ id: number; title: string; artist: string; youtube_status: string; youtube_source: string | null }, []>( + "SELECT id, title, artist, youtube_status, youtube_source FROM songs WHERE title LIKE '%&%;%' OR artist LIKE '%&%;%'", + ) + .all() + .filter((s) => HAS_ENTITY.test(s.title) || HAS_ENTITY.test(s.artist)); + for (const s of songs) { + const title = decodeEntities(s.title); + const artist = decodeEntities(s.artist); + if (title === s.title && artist === s.artist) continue; + const key = songMatchKey(title, artist); + db.query('UPDATE songs SET title = ?, artist = ?, match_key = ?, updated_at = ? WHERE id = ?').run(title, artist, key, now, s.id); + out.songs++; + if ((s.youtube_status === 'none' || s.youtube_status === 'candidate') && s.youtube_source !== 'admin') { + db.query("DELETE FROM signals WHERE song_id = ? AND kind LIKE 'yt_tried_%'").run(s.id); + out.retry++; + } + const same = db.query<{ id: number }, [string, number]>('SELECT id FROM songs WHERE match_key = ? AND id <> ? AND deleted_at IS NULL ORDER BY id').all(key, s.id); + if (same.length) out.duplicates.push({ id: s.id, title, artist, sameAs: same.map((r) => r.id) }); + } + for (const d of out.duplicates) mergeSongs(db, d.sameAs[0]!, [d.id], now); + for (const a of db.query<{ id: number; name: string }, []>("SELECT id, name FROM artists WHERE name LIKE '%&%;%'").all()) { + const name = decodeEntities(a.name); + if (name !== a.name) { + db.query('UPDATE artists SET name = ? WHERE id = ?').run(name, a.id); + out.artists++; + } + } + const rows = db + .query<{ id: number; raw_title: string; raw_artist: string }, []>( + "SELECT id, raw_title, raw_artist FROM chart_entries WHERE raw_title LIKE '%&%;%' OR raw_artist LIKE '%&%;%'", + ) + .all(); + for (const r of rows) { + const t = decodeEntities(r.raw_title); + const a = decodeEntities(r.raw_artist); + if (t === r.raw_title && a === r.raw_artist) continue; + db.query('UPDATE chart_entries SET raw_title = ?, raw_artist = ? WHERE id = ?').run(t, a, r.id); + out.chartRows++; + } + return out; + })(); +} diff --git a/apps/server/src/songs/admin-ops.ts b/apps/server/src/songs/admin-ops.ts index 2cfb29e..ef78a3d 100644 --- a/apps/server/src/songs/admin-ops.ts +++ b/apps/server/src/songs/admin-ops.ts @@ -123,7 +123,7 @@ export function mergeSongs(db: Database, target: number, others: number[], now = if (!rest.length) return 0; db.transaction(() => { for (const o of rest) { - for (const t of ['song_sources', 'song_aliases', 'song_genres', 'song_levels', 'song_artists', 'youtube_candidates', 'signals']) + for (const t of ['song_sources', 'song_aliases', 'song_genres', 'song_levels', 'song_artists', 'youtube_candidates', 'signals', 'chart_entries']) db.query(`UPDATE OR IGNORE ${t} SET song_id = ? WHERE song_id = ?`).run(target, o); db.query('UPDATE song_plays SET song_id = ? WHERE song_id = ?').run(target, o); db.query('INSERT OR IGNORE INTO lyrics (song_id, text, source, fetched_at) SELECT ?, text, source, fetched_at FROM lyrics WHERE song_id = ?').run(target, o);