곡 제목 특수문자(' &) 깨짐 수정: 멜론 차트가 16진수 엔티티를 못 풀고 norabose는 아예 안 풀던 것 → decodeEntities 한 곳에서(차트 파싱·카탈로그 저장), 이미 저장된 곡·차트 기록은 cli fix-text로 복구(키 다시 계산, 원곡 찾기 다시 시도, 깨진 키로 생긴 중복은 오래된 곡으로 합치기). 곡 합치기가 차트 기록도 옮기도록
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
/** Writes into the song catalog (docs/07). Idempotent: re-running an import never duplicates songs. */
|
||||
import type { Database } from 'bun:sqlite';
|
||||
import { songMatchKey } from '@mq/shared';
|
||||
import { decodeEntities } from './text';
|
||||
|
||||
export interface SongInput {
|
||||
melonId?: number | null;
|
||||
@@ -31,7 +32,9 @@ export class Catalog {
|
||||
}
|
||||
|
||||
/** Finds the song by Melon id, else by match key; creates it when new. Returns [id, created]. */
|
||||
upsert(s: SongInput, now = Date.now()): [number, boolean] {
|
||||
upsert(input: SongInput, now = Date.now()): [number, boolean] {
|
||||
// Sources hand us HTML-escaped text now and then (norabose `Salty & Sweet`): store what people read.
|
||||
const s = { ...input, title: decodeEntities(input.title), artists: input.artists.map((a) => ({ ...a, name: decodeEntities(a.name) })) };
|
||||
return this.db.transaction((): [number, boolean] => {
|
||||
const artist = s.artists.map((a) => a.name).join(', ');
|
||||
const key = songMatchKey(s.title, artist);
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
import type { Database } from 'bun:sqlite';
|
||||
import { songMatchKey } from '@mq/shared';
|
||||
import { Catalog, Job, sleep } from './catalog';
|
||||
import { decodeEntities } from './text';
|
||||
|
||||
export const BROWSER_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0 Safari/537.36';
|
||||
|
||||
@@ -18,15 +19,7 @@ export interface ChartRow {
|
||||
}
|
||||
|
||||
const decode = (s: string) =>
|
||||
s
|
||||
.replace(/<[^>]+>/g, '')
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'|'/g, "'")
|
||||
.replace(/&#(\d+);/g, (_, n: string) => String.fromCodePoint(Number(n)))
|
||||
decodeEntities(s.replace(/<[^>]+>/g, ''))
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
* bun apps/server/src/collector/cli.ts melon-detail [--limit N]
|
||||
* bun apps/server/src/collector/cli.ts heatmap [--limit N]
|
||||
* bun apps/server/src/collector/cli.ts levels
|
||||
* bun apps/server/src/collector/cli.ts fix-text (one-off: decode `'`/`&` saved in titles, let 원곡 찾기 retry them)
|
||||
* bun apps/server/src/collector/cli.ts stats
|
||||
* In the container: docker exec music_quiz-app-1 bun apps/server/src/collector/cli.ts …
|
||||
*/
|
||||
@@ -21,6 +22,7 @@ import { currentJob } from './catalog';
|
||||
import { collectCharts, fillMelonDetails } from './charts';
|
||||
import { collectHeatmaps } from './heatmap';
|
||||
import { computeLevels } from './levels';
|
||||
import { fixEntities } from './text';
|
||||
|
||||
const [cmd, ...rest] = process.argv.slice(2);
|
||||
const flag = (name: string) => {
|
||||
@@ -65,11 +67,14 @@ switch (cmd) {
|
||||
case 'levels':
|
||||
console.log(computeLevels(db));
|
||||
break;
|
||||
case 'fix-text':
|
||||
console.log(JSON.stringify(fixEntities(db), null, 1));
|
||||
break;
|
||||
case 'stats':
|
||||
console.log(JSON.stringify(catalogStats(db), null, 1));
|
||||
break;
|
||||
default:
|
||||
console.log('usage: cli.ts import-norabose|import-sta|match-youtube|charts|melon-detail|heatmap|levels|stats');
|
||||
console.log('usage: cli.ts import-norabose|import-sta|match-youtube|charts|melon-detail|heatmap|levels|fix-text|stats');
|
||||
process.exitCode = 1;
|
||||
}
|
||||
db.close();
|
||||
|
||||
66
apps/server/src/collector/text.test.ts
Normal file
66
apps/server/src/collector/text.test.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
/** `'` in song titles (2026-10-19): decoding on the way in and repairing what was already saved. */
|
||||
import { expect, test } from 'bun:test';
|
||||
import { join } from 'node:path';
|
||||
import { songMatchKey } from '@mq/shared';
|
||||
import { openDb } from '../db';
|
||||
import { tempDbPath } from '../test-utils';
|
||||
import { Catalog } from './catalog';
|
||||
import { parseMelon, saveChart } from './charts';
|
||||
import { decodeEntities, fixEntities } from './text';
|
||||
|
||||
test('entities: hex, decimal and named, one pass, unknown ones kept', () => {
|
||||
expect(decodeEntities('Somebody's Watching Me')).toBe("Somebody's Watching Me");
|
||||
expect(decodeEntities('AC/DC 'n' Salty & Sweet "x" <3>')).toBe(`AC/DC 'n' Salty & Sweet "x" <3>`);
|
||||
expect(decodeEntities('&#x27;')).toBe(''');
|
||||
expect(decodeEntities('R&B & Soul; &bogus; �')).toBe('R&B & Soul; &bogus; �');
|
||||
});
|
||||
|
||||
test('Melon chart rows with hex entities are stored readable', async () => {
|
||||
// the saved Melon page with the first title the way Melon escapes an apostrophe
|
||||
const html = (await Bun.file(join(import.meta.dir, 'fixtures', 'melon-chart.html')).text()).replace(/" >죄와벌 </g, '" >Somebody's Watching Me <');
|
||||
expect(parseMelon(html)[0]!.title).toBe("Somebody's Watching Me");
|
||||
// whatever the page shape, the catalog itself never stores escaped text
|
||||
const db = openDb(tempDbPath());
|
||||
const [id] = new Catalog(db).upsert({ melonId: 1, title: 'Salty & Sweet', artists: [{ name: 'Tom 'n' Jerry' }], source: 'norabose', ref: '1' });
|
||||
expect(db.query<{ title: string; artist: string; match_key: string }, [number]>('SELECT title, artist, match_key FROM songs WHERE id = ?').get(id)).toEqual({
|
||||
title: 'Salty & Sweet',
|
||||
artist: "Tom 'n' Jerry",
|
||||
match_key: songMatchKey('Salty & Sweet', "Tom 'n' Jerry"),
|
||||
});
|
||||
});
|
||||
|
||||
test('fix-text repairs saved songs, chart rows and artists, retries 원곡 찾기, merges the duplicates it uncovers', () => {
|
||||
const db = openDb(tempDbPath());
|
||||
const now = Date.now();
|
||||
const add = (title: string, artist: string, status: string, source: string | null = null) =>
|
||||
Number(
|
||||
db
|
||||
.query("INSERT INTO songs (title, artist, match_key, youtube_status, youtube_source, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 0, 0)")
|
||||
.run(title, artist, songMatchKey(title, artist), status, source).lastInsertRowid,
|
||||
);
|
||||
const broken = add('Somebody's Watching Me', 'Rockwell', 'none');
|
||||
const matched = add('I'm On Fire', 'Bruce Springsteen', 'matched', 'ytmusic');
|
||||
const admin = add('You & Me', '제니', 'candidate', 'admin');
|
||||
const original = add("I'm On Fire", 'Bruce Springsteen', 'matched', 'ytmusic');
|
||||
const clean = add('Hype Boy', 'NewJeans', 'none');
|
||||
for (const id of [broken, matched, admin, clean]) db.query("INSERT INTO signals (song_id, kind, value, fetched_at) VALUES (?, 'yt_tried_ytmusic', ?, ?)").run(id, now, now);
|
||||
db.query("INSERT INTO artists (name) VALUES ('Tom & Jerry')").run();
|
||||
db.query("INSERT INTO chart_entries (source, chart, period, rank, raw_title, raw_artist, song_id, fetched_at) VALUES ('melon', 'year-pop', '1984', 1, 'I'm On Fire', 'Bruce Springsteen', ?, 0)").run(matched);
|
||||
saveChart(db, { source: 'genie', chart: 'year', period: '1984', year: 1984 }, [{ rank: 1, ref: '1', title: 'Somebody's', artists: [{ name: 'A & B' }] }]);
|
||||
|
||||
const r = fixEntities(db);
|
||||
expect(r).toMatchObject({ songs: 3, retry: 1, artists: 1, chartRows: 2 });
|
||||
expect(r.duplicates).toEqual([{ id: matched, title: "I'm On Fire", artist: 'Bruce Springsteen', sameAs: [original] }]);
|
||||
// the copy went to the trash and its chart rows follow the song that stays
|
||||
expect(db.query<{ deleted: number }, [number]>('SELECT deleted_at IS NOT NULL deleted FROM songs WHERE id = ?').get(matched)!.deleted).toBe(1);
|
||||
const row = (id: number) => db.query<{ title: string; match_key: string }, [number]>('SELECT title, match_key FROM songs WHERE id = ?').get(id)!;
|
||||
expect(row(broken)).toEqual({ title: "Somebody's Watching Me", match_key: songMatchKey("Somebody's Watching Me", 'Rockwell') });
|
||||
expect(row(admin).title).toBe('You & Me');
|
||||
const tried = db.query<{ song_id: number }, []>("SELECT song_id FROM signals WHERE kind = 'yt_tried_ytmusic' ORDER BY song_id").all().map((x) => x.song_id);
|
||||
expect(tried).toEqual([admin, original, clean]); // only the unsettled, non-admin song is retried (the copy's mark moved over)
|
||||
expect(db.query<{ raw_title: string; raw_artist: string; song_id: number | null }, []>('SELECT raw_title, raw_artist, song_id FROM chart_entries ORDER BY source').all()).toEqual([
|
||||
{ raw_title: "Somebody's", raw_artist: 'A & B', song_id: null },
|
||||
{ raw_title: "I'm On Fire", raw_artist: 'Bruce Springsteen', song_id: original },
|
||||
]);
|
||||
expect(fixEntities(db)).toMatchObject({ songs: 0, retry: 0, artists: 0, chartRows: 0 });
|
||||
});
|
||||
69
apps/server/src/collector/text.ts
Normal file
69
apps/server/src/collector/text.ts
Normal file
@@ -0,0 +1,69 @@
|
||||
/** HTML entities in collected song text (`Somebody's`, `Salty & Sweet`): decoding and the one-off DB repair. */
|
||||
import type { Database } from 'bun:sqlite';
|
||||
import { songMatchKey } from '@mq/shared';
|
||||
import { mergeSongs } from '../songs/admin-ops';
|
||||
|
||||
const NAMED: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' };
|
||||
|
||||
/** One pass, so `&#x27;` becomes the literal text `'` (not `'`). Unknown entities stay as they are. */
|
||||
export function decodeEntities(s: string): string {
|
||||
return s.replace(/&(#[xX][0-9a-fA-F]+|#\d+|[a-zA-Z]+);/g, (m, e: string) => {
|
||||
if (e[0] !== '#') return NAMED[e.toLowerCase()] ?? m;
|
||||
const n = e[1] === 'x' || e[1] === 'X' ? parseInt(e.slice(2), 16) : Number(e.slice(1));
|
||||
return n > 0 && n <= 0x10ffff ? String.fromCodePoint(n) : m;
|
||||
});
|
||||
}
|
||||
|
||||
const HAS_ENTITY = /&(#[xX][0-9a-fA-F]+|#\d+|[a-zA-Z]+);/;
|
||||
|
||||
/**
|
||||
* Repairs songs/artists/chart rows saved with entities: decodes the text, recomputes the match key and lets
|
||||
* 원곡 찾기 try the song again (its title never matched). A song whose fixed key now equals an older song's was a
|
||||
* duplicate made by the broken key: it is merged into the oldest one (관리자 → 중복 합치기, the copy goes to the trash).
|
||||
*/
|
||||
export function fixEntities(db: Database, now = Date.now()) {
|
||||
return db.transaction(() => {
|
||||
const out = { songs: 0, retry: 0, artists: 0, chartRows: 0, duplicates: [] as { id: number; title: string; artist: string; sameAs: number[] }[] };
|
||||
const songs = db
|
||||
.query<{ id: number; title: string; artist: string; youtube_status: string; youtube_source: string | null }, []>(
|
||||
"SELECT id, title, artist, youtube_status, youtube_source FROM songs WHERE title LIKE '%&%;%' OR artist LIKE '%&%;%'",
|
||||
)
|
||||
.all()
|
||||
.filter((s) => HAS_ENTITY.test(s.title) || HAS_ENTITY.test(s.artist));
|
||||
for (const s of songs) {
|
||||
const title = decodeEntities(s.title);
|
||||
const artist = decodeEntities(s.artist);
|
||||
if (title === s.title && artist === s.artist) continue;
|
||||
const key = songMatchKey(title, artist);
|
||||
db.query('UPDATE songs SET title = ?, artist = ?, match_key = ?, updated_at = ? WHERE id = ?').run(title, artist, key, now, s.id);
|
||||
out.songs++;
|
||||
if ((s.youtube_status === 'none' || s.youtube_status === 'candidate') && s.youtube_source !== 'admin') {
|
||||
db.query("DELETE FROM signals WHERE song_id = ? AND kind LIKE 'yt_tried_%'").run(s.id);
|
||||
out.retry++;
|
||||
}
|
||||
const same = db.query<{ id: number }, [string, number]>('SELECT id FROM songs WHERE match_key = ? AND id <> ? AND deleted_at IS NULL ORDER BY id').all(key, s.id);
|
||||
if (same.length) out.duplicates.push({ id: s.id, title, artist, sameAs: same.map((r) => r.id) });
|
||||
}
|
||||
for (const d of out.duplicates) mergeSongs(db, d.sameAs[0]!, [d.id], now);
|
||||
for (const a of db.query<{ id: number; name: string }, []>("SELECT id, name FROM artists WHERE name LIKE '%&%;%'").all()) {
|
||||
const name = decodeEntities(a.name);
|
||||
if (name !== a.name) {
|
||||
db.query('UPDATE artists SET name = ? WHERE id = ?').run(name, a.id);
|
||||
out.artists++;
|
||||
}
|
||||
}
|
||||
const rows = db
|
||||
.query<{ id: number; raw_title: string; raw_artist: string }, []>(
|
||||
"SELECT id, raw_title, raw_artist FROM chart_entries WHERE raw_title LIKE '%&%;%' OR raw_artist LIKE '%&%;%'",
|
||||
)
|
||||
.all();
|
||||
for (const r of rows) {
|
||||
const t = decodeEntities(r.raw_title);
|
||||
const a = decodeEntities(r.raw_artist);
|
||||
if (t === r.raw_title && a === r.raw_artist) continue;
|
||||
db.query('UPDATE chart_entries SET raw_title = ?, raw_artist = ? WHERE id = ?').run(t, a, r.id);
|
||||
out.chartRows++;
|
||||
}
|
||||
return out;
|
||||
})();
|
||||
}
|
||||
@@ -123,7 +123,7 @@ export function mergeSongs(db: Database, target: number, others: number[], now =
|
||||
if (!rest.length) return 0;
|
||||
db.transaction(() => {
|
||||
for (const o of rest) {
|
||||
for (const t of ['song_sources', 'song_aliases', 'song_genres', 'song_levels', 'song_artists', 'youtube_candidates', 'signals'])
|
||||
for (const t of ['song_sources', 'song_aliases', 'song_genres', 'song_levels', 'song_artists', 'youtube_candidates', 'signals', 'chart_entries'])
|
||||
db.query(`UPDATE OR IGNORE ${t} SET song_id = ? WHERE song_id = ?`).run(target, o);
|
||||
db.query('UPDATE song_plays SET song_id = ? WHERE song_id = ?').run(target, o);
|
||||
db.query('INSERT OR IGNORE INTO lyrics (song_id, text, source, fetched_at) SELECT ?, text, source, fetched_at FROM lyrics WHERE song_id = ?').run(target, o);
|
||||
|
||||
Reference in New Issue
Block a user