곡 제목 특수문자(' &) 깨짐 수정: 멜론 차트가 16진수 엔티티를 못 풀고 norabose는 아예 안 풀던 것 → decodeEntities 한 곳에서(차트 파싱·카탈로그 저장), 이미 저장된 곡·차트 기록은 cli fix-text로 복구(키 다시 계산, 원곡 찾기 다시 시도, 깨진 키로 생긴 중복은 오래된 곡으로 합치기). 곡 합치기가 차트 기록도 옮기도록

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
EJClaw
2026-10-19 22:27:10 +09:00
parent 0c0b822adc
commit 127513db63
6 changed files with 148 additions and 12 deletions

View File

@@ -1,6 +1,7 @@
/** Writes into the song catalog (docs/07). Idempotent: re-running an import never duplicates songs. */
import type { Database } from 'bun:sqlite';
import { songMatchKey } from '@mq/shared';
import { decodeEntities } from './text';
export interface SongInput {
melonId?: number | null;
@@ -31,7 +32,9 @@ export class Catalog {
}
/** Finds the song by Melon id, else by match key; creates it when new. Returns [id, created]. */
upsert(s: SongInput, now = Date.now()): [number, boolean] {
upsert(input: SongInput, now = Date.now()): [number, boolean] {
// Sources hand us HTML-escaped text now and then (norabose `Salty &amp; Sweet`): store what people read.
const s = { ...input, title: decodeEntities(input.title), artists: input.artists.map((a) => ({ ...a, name: decodeEntities(a.name) })) };
return this.db.transaction((): [number, boolean] => {
const artist = s.artists.map((a) => a.name).join(', ');
const key = songMatchKey(s.title, artist);

View File

@@ -6,6 +6,7 @@
import type { Database } from 'bun:sqlite';
import { songMatchKey } from '@mq/shared';
import { Catalog, Job, sleep } from './catalog';
import { decodeEntities } from './text';
export const BROWSER_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0 Safari/537.36';
@@ -18,15 +19,7 @@ export interface ChartRow {
}
const decode = (s: string) =>
s
.replace(/<[^>]+>/g, '')
.replace(/&nbsp;/g, ' ')
.replace(/&amp;/g, '&')
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&quot;/g, '"')
.replace(/&#39;|&apos;/g, "'")
.replace(/&#(\d+);/g, (_, n: string) => String.fromCodePoint(Number(n)))
decodeEntities(s.replace(/<[^>]+>/g, ''))
.replace(/\s+/g, ' ')
.trim();

View File

@@ -7,6 +7,7 @@
* bun apps/server/src/collector/cli.ts melon-detail [--limit N]
* bun apps/server/src/collector/cli.ts heatmap [--limit N]
* bun apps/server/src/collector/cli.ts levels
* bun apps/server/src/collector/cli.ts fix-text (one-off: decode `&#x27;`/`&amp;` saved in titles, let 원곡 찾기 retry them)
* bun apps/server/src/collector/cli.ts stats
* In the container: docker exec music_quiz-app-1 bun apps/server/src/collector/cli.ts …
*/
@@ -21,6 +22,7 @@ import { currentJob } from './catalog';
import { collectCharts, fillMelonDetails } from './charts';
import { collectHeatmaps } from './heatmap';
import { computeLevels } from './levels';
import { fixEntities } from './text';
const [cmd, ...rest] = process.argv.slice(2);
const flag = (name: string) => {
@@ -65,11 +67,14 @@ switch (cmd) {
case 'levels':
console.log(computeLevels(db));
break;
case 'fix-text':
console.log(JSON.stringify(fixEntities(db), null, 1));
break;
case 'stats':
console.log(JSON.stringify(catalogStats(db), null, 1));
break;
default:
console.log('usage: cli.ts import-norabose|import-sta|match-youtube|charts|melon-detail|heatmap|levels|stats');
console.log('usage: cli.ts import-norabose|import-sta|match-youtube|charts|melon-detail|heatmap|levels|fix-text|stats');
process.exitCode = 1;
}
db.close();

View File

@@ -0,0 +1,66 @@
/** `&#x27;` in song titles (2026-10-19): decoding on the way in and repairing what was already saved. */
import { expect, test } from 'bun:test';
import { join } from 'node:path';
import { songMatchKey } from '@mq/shared';
import { openDb } from '../db';
import { tempDbPath } from '../test-utils';
import { Catalog } from './catalog';
import { parseMelon, saveChart } from './charts';
import { decodeEntities, fixEntities } from './text';
test('entities: hex, decimal and named, one pass, unknown ones kept', () => {
expect(decodeEntities('Somebody&#x27;s Watching Me')).toBe("Somebody's Watching Me");
expect(decodeEntities('AC&#x2F;DC &#39;n&#039; Salty &amp; Sweet &quot;x&quot; &lt;3&gt;')).toBe(`AC/DC 'n' Salty & Sweet "x" <3>`);
expect(decodeEntities('&amp;#x27;')).toBe('&#x27;');
expect(decodeEntities('R&B & Soul; &bogus; &#0;')).toBe('R&B & Soul; &bogus; &#0;');
});
test('Melon chart rows with hex entities are stored readable', async () => {
// the saved Melon page with the first title the way Melon escapes an apostrophe
const html = (await Bun.file(join(import.meta.dir, 'fixtures', 'melon-chart.html')).text()).replace(/" >죄와벌 </g, '" >Somebody&#x27;s Watching Me <');
expect(parseMelon(html)[0]!.title).toBe("Somebody's Watching Me");
// whatever the page shape, the catalog itself never stores escaped text
const db = openDb(tempDbPath());
const [id] = new Catalog(db).upsert({ melonId: 1, title: 'Salty &amp; Sweet', artists: [{ name: 'Tom &#x27;n&#x27; Jerry' }], source: 'norabose', ref: '1' });
expect(db.query<{ title: string; artist: string; match_key: string }, [number]>('SELECT title, artist, match_key FROM songs WHERE id = ?').get(id)).toEqual({
title: 'Salty & Sweet',
artist: "Tom 'n' Jerry",
match_key: songMatchKey('Salty & Sweet', "Tom 'n' Jerry"),
});
});
test('fix-text repairs saved songs, chart rows and artists, retries 원곡 찾기, merges the duplicates it uncovers', () => {
const db = openDb(tempDbPath());
const now = Date.now();
const add = (title: string, artist: string, status: string, source: string | null = null) =>
Number(
db
.query("INSERT INTO songs (title, artist, match_key, youtube_status, youtube_source, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 0, 0)")
.run(title, artist, songMatchKey(title, artist), status, source).lastInsertRowid,
);
const broken = add('Somebody&#x27;s Watching Me', 'Rockwell', 'none');
const matched = add('I&#x27;m On Fire', 'Bruce Springsteen', 'matched', 'ytmusic');
const admin = add('You &amp; Me', '제니', 'candidate', 'admin');
const original = add("I'm On Fire", 'Bruce Springsteen', 'matched', 'ytmusic');
const clean = add('Hype Boy', 'NewJeans', 'none');
for (const id of [broken, matched, admin, clean]) db.query("INSERT INTO signals (song_id, kind, value, fetched_at) VALUES (?, 'yt_tried_ytmusic', ?, ?)").run(id, now, now);
db.query("INSERT INTO artists (name) VALUES ('Tom &amp; Jerry')").run();
db.query("INSERT INTO chart_entries (source, chart, period, rank, raw_title, raw_artist, song_id, fetched_at) VALUES ('melon', 'year-pop', '1984', 1, 'I&#x27;m On Fire', 'Bruce Springsteen', ?, 0)").run(matched);
saveChart(db, { source: 'genie', chart: 'year', period: '1984', year: 1984 }, [{ rank: 1, ref: '1', title: 'Somebody&#x27;s', artists: [{ name: 'A &amp; B' }] }]);
const r = fixEntities(db);
expect(r).toMatchObject({ songs: 3, retry: 1, artists: 1, chartRows: 2 });
expect(r.duplicates).toEqual([{ id: matched, title: "I'm On Fire", artist: 'Bruce Springsteen', sameAs: [original] }]);
// the copy went to the trash and its chart rows follow the song that stays
expect(db.query<{ deleted: number }, [number]>('SELECT deleted_at IS NOT NULL deleted FROM songs WHERE id = ?').get(matched)!.deleted).toBe(1);
const row = (id: number) => db.query<{ title: string; match_key: string }, [number]>('SELECT title, match_key FROM songs WHERE id = ?').get(id)!;
expect(row(broken)).toEqual({ title: "Somebody's Watching Me", match_key: songMatchKey("Somebody's Watching Me", 'Rockwell') });
expect(row(admin).title).toBe('You & Me');
const tried = db.query<{ song_id: number }, []>("SELECT song_id FROM signals WHERE kind = 'yt_tried_ytmusic' ORDER BY song_id").all().map((x) => x.song_id);
expect(tried).toEqual([admin, original, clean]); // only the unsettled, non-admin song is retried (the copy's mark moved over)
expect(db.query<{ raw_title: string; raw_artist: string; song_id: number | null }, []>('SELECT raw_title, raw_artist, song_id FROM chart_entries ORDER BY source').all()).toEqual([
{ raw_title: "Somebody's", raw_artist: 'A & B', song_id: null },
{ raw_title: "I'm On Fire", raw_artist: 'Bruce Springsteen', song_id: original },
]);
expect(fixEntities(db)).toMatchObject({ songs: 0, retry: 0, artists: 0, chartRows: 0 });
});

View File

@@ -0,0 +1,69 @@
/** HTML entities in collected song text (`Somebody&#x27;s`, `Salty &amp; Sweet`): decoding and the one-off DB repair. */
import type { Database } from 'bun:sqlite';
import { songMatchKey } from '@mq/shared';
import { mergeSongs } from '../songs/admin-ops';
const NAMED: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' };
/** One pass, so `&amp;#x27;` becomes the literal text `&#x27;` (not `'`). Unknown entities stay as they are. */
export function decodeEntities(s: string): string {
return s.replace(/&(#[xX][0-9a-fA-F]+|#\d+|[a-zA-Z]+);/g, (m, e: string) => {
if (e[0] !== '#') return NAMED[e.toLowerCase()] ?? m;
const n = e[1] === 'x' || e[1] === 'X' ? parseInt(e.slice(2), 16) : Number(e.slice(1));
return n > 0 && n <= 0x10ffff ? String.fromCodePoint(n) : m;
});
}
const HAS_ENTITY = /&(#[xX][0-9a-fA-F]+|#\d+|[a-zA-Z]+);/;
/**
* Repairs songs/artists/chart rows saved with entities: decodes the text, recomputes the match key and lets
* 원곡 찾기 try the song again (its title never matched). A song whose fixed key now equals an older song's was a
* duplicate made by the broken key: it is merged into the oldest one (관리자 → 중복 합치기, the copy goes to the trash).
*/
export function fixEntities(db: Database, now = Date.now()) {
return db.transaction(() => {
const out = { songs: 0, retry: 0, artists: 0, chartRows: 0, duplicates: [] as { id: number; title: string; artist: string; sameAs: number[] }[] };
const songs = db
.query<{ id: number; title: string; artist: string; youtube_status: string; youtube_source: string | null }, []>(
"SELECT id, title, artist, youtube_status, youtube_source FROM songs WHERE title LIKE '%&%;%' OR artist LIKE '%&%;%'",
)
.all()
.filter((s) => HAS_ENTITY.test(s.title) || HAS_ENTITY.test(s.artist));
for (const s of songs) {
const title = decodeEntities(s.title);
const artist = decodeEntities(s.artist);
if (title === s.title && artist === s.artist) continue;
const key = songMatchKey(title, artist);
db.query('UPDATE songs SET title = ?, artist = ?, match_key = ?, updated_at = ? WHERE id = ?').run(title, artist, key, now, s.id);
out.songs++;
if ((s.youtube_status === 'none' || s.youtube_status === 'candidate') && s.youtube_source !== 'admin') {
db.query("DELETE FROM signals WHERE song_id = ? AND kind LIKE 'yt_tried_%'").run(s.id);
out.retry++;
}
const same = db.query<{ id: number }, [string, number]>('SELECT id FROM songs WHERE match_key = ? AND id <> ? AND deleted_at IS NULL ORDER BY id').all(key, s.id);
if (same.length) out.duplicates.push({ id: s.id, title, artist, sameAs: same.map((r) => r.id) });
}
for (const d of out.duplicates) mergeSongs(db, d.sameAs[0]!, [d.id], now);
for (const a of db.query<{ id: number; name: string }, []>("SELECT id, name FROM artists WHERE name LIKE '%&%;%'").all()) {
const name = decodeEntities(a.name);
if (name !== a.name) {
db.query('UPDATE artists SET name = ? WHERE id = ?').run(name, a.id);
out.artists++;
}
}
const rows = db
.query<{ id: number; raw_title: string; raw_artist: string }, []>(
"SELECT id, raw_title, raw_artist FROM chart_entries WHERE raw_title LIKE '%&%;%' OR raw_artist LIKE '%&%;%'",
)
.all();
for (const r of rows) {
const t = decodeEntities(r.raw_title);
const a = decodeEntities(r.raw_artist);
if (t === r.raw_title && a === r.raw_artist) continue;
db.query('UPDATE chart_entries SET raw_title = ?, raw_artist = ? WHERE id = ?').run(t, a, r.id);
out.chartRows++;
}
return out;
})();
}

View File

@@ -123,7 +123,7 @@ export function mergeSongs(db: Database, target: number, others: number[], now =
if (!rest.length) return 0;
db.transaction(() => {
for (const o of rest) {
for (const t of ['song_sources', 'song_aliases', 'song_genres', 'song_levels', 'song_artists', 'youtube_candidates', 'signals'])
for (const t of ['song_sources', 'song_aliases', 'song_genres', 'song_levels', 'song_artists', 'youtube_candidates', 'signals', 'chart_entries'])
db.query(`UPDATE OR IGNORE ${t} SET song_id = ? WHERE song_id = ?`).run(target, o);
db.query('UPDATE song_plays SET song_id = ? WHERE song_id = ?').run(target, o);
db.query('INSERT OR IGNORE INTO lyrics (song_id, text, source, fetched_at) SELECT ?, text, source, fetched_at FROM lyrics WHERE song_id = ?').run(target, o);