원곡찾기 후보 자동 확정: 가수·제목 비교 오류 수정(라틴 악센트 JAŸ-Z=JAY-Z, '및'·with 구분, 앞 The 무시, 'X JAPAN'을 x로 자르던 것), 'Radio Edit/Original Version'은 같은 녹음으로, 'ver.' 뒤 괄호에서 못 잡던 버전 표시 수정. 이미 확정된 곡에서 2곡 이상 확인된 한글/영문 가수 이름(태연=TAEYEON)을 배워 씀(레이블·컴필레이션 제외). 길이 15초 넘게 다르면 자동 확정 안 함. cli reconfirm-youtube(저장된 검색 결과로 다시 비교, --dry-run). 관리자 원곡 찾기 목록에 채널 이름 대신 실제 가수와 확정 안 된 이유('가수 다름: Ashiyane', '길이 40초 차이')
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
52
apps/server/src/collector/artist-aliases.ts
Normal file
52
apps/server/src/collector/artist-aliases.ts
Normal file
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* Learns spellings of the same artist across Melon and YouTube Music ("태연" ↔ "TAEYEON") from songs that are already
|
||||
* confirmed (matched) with a YouTube Music result. A pair is only trusted once it was seen on 2+ different songs.
|
||||
*/
|
||||
import type { Database } from 'bun:sqlite';
|
||||
import { compareKey } from '@mq/shared';
|
||||
import { artistsMatch, type ArtistAliases } from './score';
|
||||
|
||||
/** Labels, not artists: never learnt as someone's other name. */
|
||||
const GENERIC = new Set(['variousartists', 'various', 'va', 'release', 'unknown', 'unknownartist', '여러아티스트', '다양한아티스트', 'smtown']);
|
||||
const MULTI = /,|&|\/|×|\s및\s|\bwith\b|\bfeat\.?|\bft\./i;
|
||||
|
||||
/** One artist's key, or null for duets/groups-with-guests. The Melon side drops a "(…)" tail, the YouTube side keeps it ("f(x)"). */
|
||||
function single(artist: string, dropTail: boolean): string | null {
|
||||
if (MULTI.test(artist)) return null;
|
||||
const k = compareKey(dropTail ? artist.replace(/\s*\([^)]*\)\s*$/, '') : artist);
|
||||
return k && !GENERIC.has(k) ? k : null;
|
||||
}
|
||||
|
||||
export function learnArtistAliases(db: Database, minSongs = 2): ArtistAliases {
|
||||
const rows = db
|
||||
.query<{ id: number; artist: string; yt: string }, []>(
|
||||
`SELECT s.id, s.artist, c.channel yt FROM songs s
|
||||
JOIN youtube_candidates c ON c.song_id = s.id AND c.video_id = s.youtube_id AND c.source = 'ytmusic'
|
||||
WHERE s.youtube_status = 'matched' AND s.deleted_at IS NULL AND c.channel IS NOT NULL`,
|
||||
)
|
||||
.all();
|
||||
const seen = new Map<string, Set<number>>();
|
||||
for (const r of rows) {
|
||||
if (artistsMatch(r.artist, r.yt)) continue;
|
||||
const a = single(r.artist, true);
|
||||
const b = single(r.yt, false);
|
||||
if (!a || !b) continue;
|
||||
const key = `${a}\t${b}`;
|
||||
let ids = seen.get(key);
|
||||
if (!ids) seen.set(key, (ids = new Set()));
|
||||
ids.add(r.id);
|
||||
}
|
||||
// A YouTube name claimed by 3+ different Melon artists is a label or a compilation, not a spelling.
|
||||
const claims = new Map<string, number>();
|
||||
for (const [key, ids] of seen) if (ids.size >= minSongs) claims.set(key.split('\t')[1]!, (claims.get(key.split('\t')[1]!) ?? 0) + 1);
|
||||
const out: ArtistAliases = new Map();
|
||||
for (const [key, ids] of seen) {
|
||||
if (ids.size < minSongs) continue;
|
||||
const [a, b] = key.split('\t') as [string, string];
|
||||
if ((claims.get(b) ?? 0) >= 3) continue;
|
||||
let set = out.get(a);
|
||||
if (!set) out.set(a, (set = new Set()));
|
||||
set.add(b);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -3,6 +3,7 @@
|
||||
* bun apps/server/src/collector/cli.ts import-norabose [--limit N]
|
||||
* bun apps/server/src/collector/cli.ts import-sta
|
||||
* bun apps/server/src/collector/cli.ts match-youtube [--limit N] [--delay ms] [--deadline epochMs] [--require-cookie] (default pace: 2s apart; cron caps a day at 3000; most popular level first)
|
||||
* bun apps/server/src/collector/cli.ts reconfirm-youtube [--dry-run] [--delay ms] (후보 곡을 이미 저장된 검색 결과로 현재 기준에 다시 비교해 확정)
|
||||
* bun apps/server/src/collector/cli.ts charts [--from YYYY] [--to YYYY] [--sources melon-ye,genie-ye,tj-ye,melon-rt,bugs-rt] [--refetch]
|
||||
* bun apps/server/src/collector/cli.ts melon-detail [--limit N]
|
||||
* bun apps/server/src/collector/cli.ts heatmap [--limit N]
|
||||
@@ -17,7 +18,7 @@ import { join } from 'node:path';
|
||||
import { openDb } from '../db';
|
||||
import { importNorabose } from './norabose';
|
||||
import { importSta } from './sta';
|
||||
import { matchYoutube } from './match-youtube';
|
||||
import { matchYoutube, reconfirmCandidates } from './match-youtube';
|
||||
import { pickSearcher } from './ytmusic';
|
||||
import { catalogStats } from '../songs/store';
|
||||
import { currentJob } from './catalog';
|
||||
@@ -58,6 +59,9 @@ switch (cmd) {
|
||||
console.log(await matchYoutube(db, searcher, { limit: flag('limit'), delayMs: flag('delay'), deadline: flag('deadline'), source: searcher.kind }));
|
||||
break;
|
||||
}
|
||||
case 'reconfirm-youtube':
|
||||
console.log(await reconfirmCandidates(db, { dryRun: rest.includes('--dry-run'), delayMs: flag('delay'), log: console.log }));
|
||||
break;
|
||||
case 'charts': {
|
||||
const i = rest.indexOf('--sources');
|
||||
console.log(await collectCharts(db, { from: flag('from'), to: flag('to'), sources: i >= 0 ? rest[i + 1]!.split(',') : undefined, refetch: rest.includes('--refetch'), delayMs: flag('delay') }));
|
||||
@@ -85,7 +89,7 @@ switch (cmd) {
|
||||
console.log(JSON.stringify(catalogStats(db), null, 1));
|
||||
break;
|
||||
default:
|
||||
console.log('usage: cli.ts import-norabose|import-sta|match-youtube|charts|melon-detail|heatmap|loudness|age-audio|levels|fix-text|stats');
|
||||
console.log('usage: cli.ts import-norabose|import-sta|match-youtube|reconfirm-youtube|charts|melon-detail|heatmap|loudness|age-audio|levels|fix-text|stats');
|
||||
process.exitCode = 1;
|
||||
}
|
||||
db.close();
|
||||
|
||||
@@ -5,8 +5,9 @@ import { openDb } from '../db';
|
||||
import { Catalog } from './catalog';
|
||||
import { genresOf, provisionalLevel } from './norabose';
|
||||
import { importSta } from './sta';
|
||||
import { matchYoutube } from './match-youtube';
|
||||
import { isConfident, scoreResult } from './score';
|
||||
import { matchYoutube, reconfirmCandidates } from './match-youtube';
|
||||
import { isConfident, notConfidentReasons, scoreResult } from './score';
|
||||
import { learnArtistAliases } from './artist-aliases';
|
||||
import { parseDuration, parseSearch, type YtmSong } from './ytmusic';
|
||||
import { catalogStats, listSongs } from '../songs/store';
|
||||
import { tempDbPath } from '../test-utils';
|
||||
@@ -46,6 +47,53 @@ test('scoring: title+artist confident, MR/cover not, wrong length not', () => {
|
||||
expect(isConfident(scoreResult(song, yt('다른 노래', 'SG워너비', 237), 0))).toBe(false);
|
||||
});
|
||||
|
||||
test('scoring: accents, 및, "Radio Edit Version" and the length limit', () => {
|
||||
const yt = (title: string, artists: string, durationSec: number | null): YtmSong => ({ videoId: 'x', title, artists, album: null, durationSec });
|
||||
expect(isConfident(scoreResult({ title: 'Empire State Of Mind', artist: 'JAY-Z', durationSec: null }, yt('Empire State Of Mind', 'JAŸ-Z 및 Alicia Keys', 277), 0))).toBe(true);
|
||||
expect(isConfident(scoreResult({ title: 'Forever Love', artist: 'X JAPAN, YOSHIKI', durationSec: null }, yt('Forever Love', 'X JAPAN 및 YOSHIKI', 300), 0))).toBe(true);
|
||||
expect(isConfident(scoreResult({ title: 'Set Me On Fire', artist: 'Flashlight Brown', durationSec: 180 }, yt('Set Me On Fire (Radio Edit Version)', 'Flashlight Brown', 182), 0))).toBe(true);
|
||||
expect(isConfident(scoreResult({ title: '심장이 없어', artist: '에이트', durationSec: 208 }, yt('심장이 없어 (Rough Ver.)', '에이트', 208), 0))).toBe(false);
|
||||
// title + artist but 18 seconds off: never auto-confirmed
|
||||
const off = scoreResult({ title: 'Human', artist: 'The Human League', durationSec: 248 }, yt('Human', 'Human League', 266), 0);
|
||||
expect(isConfident(off)).toBe(false);
|
||||
expect(notConfidentReasons(off)).toEqual(['길이 18초 차이']);
|
||||
const other = scoreResult({ title: 'Flower Dance', artist: 'DJ Okawari', durationSec: null }, yt('Flower Dance', 'Ashiyane', 277), 0);
|
||||
expect(notConfidentReasons(other)).toEqual(['가수 다름: Ashiyane']);
|
||||
});
|
||||
|
||||
test('artist names learnt from confirmed songs (2+ songs) confirm waiting candidates', async () => {
|
||||
const db = freshDb();
|
||||
const c = new Catalog(db);
|
||||
const confirmed = (title: string, artist: string, ytArtist: string, video: string) => {
|
||||
const [id] = c.upsert({ title, artists: [{ name: artist }], durationSec: 200, source: 't', ref: video });
|
||||
db.query("UPDATE songs SET youtube_id = ?, youtube_status = 'matched', youtube_source = 'ytmusic' WHERE id = ?").run(video, id);
|
||||
db.query("INSERT INTO youtube_candidates (song_id, video_id, title, channel, duration_sec, score, source) VALUES (?, ?, ?, ?, 200, 70, 'ytmusic')").run(id, video, title, ytArtist);
|
||||
};
|
||||
confirmed('I', '태연', 'TAEYEON', 'v1');
|
||||
confirmed('Rain', '태연', 'TAEYEON', 'v2');
|
||||
confirmed('Only', '한번만', 'ONCE', 'v3'); // seen once: not trusted
|
||||
expect([...(learnArtistAliases(db).get('태연') ?? [])]).toEqual(['taeyeon']);
|
||||
expect(learnArtistAliases(db).has('한번만')).toBe(false);
|
||||
const waiting = (title: string, artist: string, dur: number | null, cand: { video: string; title: string; artist: string; dur: number }) => {
|
||||
const [id] = c.upsert({ title, artists: [{ name: artist }], durationSec: dur, source: 't', ref: `w${title}` });
|
||||
db.query("UPDATE songs SET youtube_id = 'fan', youtube_status = 'candidate', youtube_source = 'norabose' WHERE id = ?").run(id);
|
||||
db.query("INSERT INTO youtube_candidates (song_id, video_id, title, channel, duration_sec, score, source) VALUES (?, ?, ?, ?, ?, 50, 'ytmusic')").run(id, cand.video, cand.title, cand.artist, cand.dur);
|
||||
return id!;
|
||||
};
|
||||
const happy = waiting('Happy', '태연', 220, { video: 'happyvid001', title: 'Happy', artist: 'TAEYEON', dur: 225 });
|
||||
const flower = waiting('Flower Dance', 'DJ Okawari', null, { video: 'ashiyane001', title: 'Flower Dance', artist: 'Ashiyane', dur: 277 });
|
||||
const far = waiting('Weekend', '태연', 230, { video: 'weekend0001', title: 'Weekend', artist: 'TAEYEON', dur: 260 });
|
||||
const dry = await reconfirmCandidates(db, { dryRun: true });
|
||||
expect(dry.matched).toBe(1);
|
||||
expect(db.query<{ n: number }, []>("SELECT COUNT(*) n FROM songs WHERE youtube_status = 'candidate'").get()!.n).toBe(3); // dry run writes nothing
|
||||
const stats = await reconfirmCandidates(db, { delayMs: 0 }, async (id) => ({ state: 'ok', author: 'TAEYEON - Topic', title: id }));
|
||||
expect(stats.matched).toBe(1);
|
||||
const st = (id: number) => db.query<{ youtube_status: string; youtube_id: string }, [number]>('SELECT youtube_status, youtube_id FROM songs WHERE id = ?').get(id)!;
|
||||
expect(st(happy)).toEqual({ youtube_status: 'matched', youtube_id: 'happyvid001' });
|
||||
expect(st(flower).youtube_status).toBe('candidate');
|
||||
expect(st(far).youtube_status).toBe('candidate');
|
||||
});
|
||||
|
||||
function freshDb() {
|
||||
return openDb(tempDbPath());
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import type { Database } from 'bun:sqlite';
|
||||
import { Job, sleep } from './catalog';
|
||||
import { checkEmbed } from './oembed';
|
||||
import { isConfident, scoreResult, type Scored } from './score';
|
||||
import { learnArtistAliases } from './artist-aliases';
|
||||
import { CookieError, type YtMusic, type YtmSong } from './ytmusic';
|
||||
|
||||
interface Row {
|
||||
@@ -27,6 +28,19 @@ export interface MatchOptions {
|
||||
log?: (s: string) => void;
|
||||
}
|
||||
|
||||
/** Writes an automatic result only while the song is still open for automatic matching (no admin choice meanwhile). */
|
||||
function setAutoVideo(db: Database, id: number, status: 'matched' | 'candidate', v: { videoId: string; author: string; title: string }, source: string): boolean {
|
||||
return (
|
||||
db
|
||||
.query(
|
||||
`UPDATE songs SET youtube_id = ?, youtube_status = ?, youtube_source = ?, youtube_channel = ?, youtube_title = ?, embeddable = 1,
|
||||
youtube_checked_at = ?, updated_at = ?
|
||||
WHERE id = ? AND COALESCE(youtube_source, '') <> 'admin' AND youtube_status IN ('none','candidate')`,
|
||||
)
|
||||
.run(v.videoId, status, source, v.author, v.title, Date.now(), Date.now(), id).changes === 1
|
||||
);
|
||||
}
|
||||
|
||||
export async function matchYoutube(db: Database, ytm: Pick<YtMusic, 'searchSongs'>, opts: MatchOptions = {}, embed = checkEmbed) {
|
||||
const retryBefore = Date.now() - (opts.retryDays ?? 7) * 86400_000;
|
||||
// Tries are remembered per search method: songs the weaker fallback could not settle are retried once the cookie works.
|
||||
@@ -45,20 +59,14 @@ export async function matchYoutube(db: Database, ytm: Pick<YtMusic, 'searchSongs
|
||||
const stats = { matched: 0, candidate: 0, none: 0 };
|
||||
let tried = 0;
|
||||
const src = opts.source ?? 'ytmusic';
|
||||
const aliases = learnArtistAliases(db);
|
||||
// 관리자 → 원곡 찾기 reads the deadline back from this line
|
||||
job.log(`start ${rows.length} songs via ${src}${opts.deadline !== undefined ? ` deadline ${new Date(opts.deadline).toISOString()}` : ''}`);
|
||||
/**
|
||||
* Writes the result only if the song is still open for automatic matching: an admin may have saved a video
|
||||
* while we were waiting on the search. Returns whether the row was changed.
|
||||
*/
|
||||
const setVideo = (id: number, status: 'matched' | 'candidate', v: { videoId: string; author: string; title: string }, source: string) =>
|
||||
db
|
||||
.query(
|
||||
`UPDATE songs SET youtube_id = ?, youtube_status = ?, youtube_source = ?, youtube_channel = ?, youtube_title = ?, embeddable = 1,
|
||||
youtube_checked_at = ?, updated_at = ?
|
||||
WHERE id = ? AND COALESCE(youtube_source, '') <> 'admin' AND youtube_status IN ('none','candidate')`,
|
||||
)
|
||||
.run(v.videoId, status, source, v.author, v.title, Date.now(), Date.now(), id).changes === 1;
|
||||
const setVideo = (id: number, status: 'matched' | 'candidate', v: { videoId: string; author: string; title: string }, source: string) => setAutoVideo(db, id, status, v, source);
|
||||
const adminOwned = (id: number) =>
|
||||
db.query<{ n: number }, [number]>("SELECT COUNT(*) n FROM songs WHERE id = ? AND COALESCE(youtube_source, '') = 'admin'").get(id)!.n > 0;
|
||||
try {
|
||||
@@ -77,7 +85,7 @@ export async function matchYoutube(db: Database, ytm: Pick<YtMusic, 'searchSongs
|
||||
await sleep(60_000);
|
||||
results = await ytm.searchSongs(query);
|
||||
}
|
||||
const scored: Scored[] = results.slice(0, 8).map((r, rank) => scoreResult({ title: s.title, artist: s.artist, durationSec: s.duration_sec }, r, rank));
|
||||
const scored: Scored[] = results.slice(0, 8).map((r, rank) => scoreResult({ title: s.title, artist: s.artist, durationSec: s.duration_sec }, r, rank, aliases));
|
||||
scored.sort((a, b) => b.score - a.score);
|
||||
for (const c of scored.slice(0, 5)) {
|
||||
db.query(
|
||||
@@ -129,3 +137,59 @@ export async function matchYoutube(db: Database, ytm: Pick<YtMusic, 'searchSongs
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-checks songs left as 후보 against the YouTube Music results already saved for them, with the current rules
|
||||
* (accent-insensitive names, 및/with separators, learnt Korean/English artist names, benign "Radio Edit Version").
|
||||
* No search: only the embed check per confirmed video. `dryRun` counts without writing or calling YouTube.
|
||||
*/
|
||||
export async function reconfirmCandidates(db: Database, opts: { dryRun?: boolean; delayMs?: number; log?: (s: string) => void } = {}, embed = checkEmbed) {
|
||||
const aliases = learnArtistAliases(db);
|
||||
const songs = db
|
||||
.query<Row, []>(
|
||||
`SELECT id, title, artist, duration_sec FROM songs
|
||||
WHERE deleted_at IS NULL AND youtube_status = 'candidate' AND COALESCE(youtube_source, '') <> 'admin' ORDER BY id`,
|
||||
)
|
||||
.all();
|
||||
const cands = db.query<{ video_id: string; title: string | null; channel: string | null; duration_sec: number | null }, [number]>(
|
||||
"SELECT video_id, title, channel, duration_sec FROM youtube_candidates WHERE song_id = ? AND source = 'ytmusic' ORDER BY score DESC",
|
||||
);
|
||||
const job = opts.dryRun ? null : new Job(db, 'reconfirm-youtube', songs.length);
|
||||
job?.log(`start ${songs.length} candidate songs, ${aliases.size} learnt artist names`);
|
||||
const stats = { checked: songs.length, matched: 0, unplayable: 0, aliases: aliases.size };
|
||||
try {
|
||||
for (const [i, s] of songs.entries()) {
|
||||
const scored = cands
|
||||
.all(s.id)
|
||||
.map((c) => scoreResult({ title: s.title, artist: s.artist, durationSec: s.duration_sec }, { videoId: c.video_id, title: c.title ?? '', artists: c.channel ?? '', album: null, durationSec: c.duration_sec }, -1, aliases))
|
||||
.filter(isConfident)
|
||||
.sort((a, b) => b.score - a.score);
|
||||
for (const c of scored) {
|
||||
if (opts.dryRun) {
|
||||
stats.matched++;
|
||||
opts.log?.(`${s.id}\t${s.artist} - ${s.title}\t→ ${c.artists} - ${c.title} (${c.lengthDiff ?? '?'}s)`);
|
||||
break;
|
||||
}
|
||||
const e = await embed(c.videoId);
|
||||
await sleep(opts.delayMs ?? 300);
|
||||
if (e.state !== 'ok') {
|
||||
stats.unplayable++;
|
||||
continue;
|
||||
}
|
||||
if (setAutoVideo(db, s.id, 'matched', { videoId: c.videoId, author: e.author, title: e.title }, 'ytmusic')) {
|
||||
if (!s.duration_sec && c.durationSec) db.query('UPDATE songs SET duration_sec = ? WHERE id = ?').run(c.durationSec, s.id);
|
||||
stats.matched++;
|
||||
}
|
||||
break;
|
||||
}
|
||||
job?.progress(i + 1);
|
||||
}
|
||||
job?.log(`reconfirm done: matched ${stats.matched} of ${songs.length}`);
|
||||
job?.finish('done');
|
||||
} catch (e) {
|
||||
job?.log(`stopped: ${(e as Error).message}`);
|
||||
job?.finish('failed');
|
||||
throw e;
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
|
||||
@@ -2,35 +2,74 @@
|
||||
import { artistKeys, keysOverlap, titleKeys } from '@mq/shared';
|
||||
import type { YtmSong } from './ytmusic';
|
||||
|
||||
const JUNK = /\b(inst|instrumental|mr|karaoke|cover|remix|sped ?up|slowed|live|ver\.|version|acoustic)\b|노래방|커버|라이브|반주/i;
|
||||
const JUNK = /\b(inst|instrumental|mr|karaoke|cover|remix|sped ?up|slowed|live|ver|version|acoustic|made famous by|in the style of|originally performed|tribute)\b|노래방|커버|라이브|반주/i;
|
||||
/** "Radio Edit Version", "Original Version", "7\" Version" are the same recording, not a different take. */
|
||||
const BENIGN = /\b(?:radio edit|original|album|single|mono|stereo|remaster(?:ed)?)\s*(?:ver\.?|version)?|\d{1,2}"\s*(?:ver\.?|version)|\bedit\b/gi;
|
||||
/** Auto-confirm only within this length difference (when both lengths are known). */
|
||||
export const MAX_LENGTH_DIFF = 15;
|
||||
|
||||
/** Song-side artist key → YouTube-side keys learnt from already confirmed songs ("태연" → "taeyeon"). */
|
||||
export type ArtistAliases = Map<string, Set<string>>;
|
||||
|
||||
export interface Scored extends YtmSong {
|
||||
score: number;
|
||||
titleOk: boolean;
|
||||
artistOk: boolean;
|
||||
junk: boolean;
|
||||
/** |song length − result length| in seconds, null when either is unknown. */
|
||||
lengthDiff: number | null;
|
||||
}
|
||||
|
||||
export function scoreResult(song: { title: string; artist: string; durationSec: number | null }, r: YtmSong, rank: number): Scored {
|
||||
export function isJunkTitle(title: string): boolean {
|
||||
return JUNK.test(title.replace(BENIGN, ' '));
|
||||
}
|
||||
|
||||
export function artistsMatch(songArtist: string, resultArtists: string, aliases?: ArtistAliases): boolean {
|
||||
const a = artistKeys(songArtist);
|
||||
const b = artistKeys(resultArtists);
|
||||
if (keysOverlap(a, b)) return true;
|
||||
return !!aliases && a.some((k) => b.some((x) => aliases.get(k)?.has(x)));
|
||||
}
|
||||
|
||||
export function scoreResult(song: { title: string; artist: string; durationSec: number | null }, r: YtmSong, rank: number, aliases?: ArtistAliases): Scored {
|
||||
const titleOk = keysOverlap(titleKeys(song.title), titleKeys(r.title));
|
||||
const artistOk = keysOverlap(artistKeys(song.artist), artistKeys(r.artists));
|
||||
const artistOk = artistsMatch(song.artist, r.artists, aliases);
|
||||
const lengthDiff = song.durationSec && r.durationSec ? Math.abs(song.durationSec - r.durationSec) : null;
|
||||
let score = 0;
|
||||
if (titleOk) score += 50;
|
||||
if (artistOk) score += 30;
|
||||
if (song.durationSec && r.durationSec) {
|
||||
const d = Math.abs(song.durationSec - r.durationSec);
|
||||
if (d <= 3) score += 20;
|
||||
else if (d <= 8) score += 10;
|
||||
else if (d > 20) score -= 30;
|
||||
if (lengthDiff !== null) {
|
||||
if (lengthDiff <= 3) score += 20;
|
||||
else if (lengthDiff <= 8) score += 10;
|
||||
else if (lengthDiff > 20) score -= 30;
|
||||
}
|
||||
if (rank === 0) score += 5;
|
||||
// Plain YouTube fallback: only official Topic art tracks are trusted; MVs, lyric videos and fan uploads are scored down.
|
||||
if (r.topic === false) score -= 25;
|
||||
// Junk words only count when the original title doesn't have them (a real "Remix" song stays fine).
|
||||
if (JUNK.test(r.title) && !JUNK.test(song.title)) score -= 40;
|
||||
return { ...r, score, titleOk, artistOk };
|
||||
const junk = isJunkTitle(r.title) && !isJunkTitle(song.title);
|
||||
if (junk) score -= 40;
|
||||
return { ...r, score, titleOk, artistOk, junk, lengthDiff };
|
||||
}
|
||||
|
||||
/** Auto-confirm: title match plus artist (80) or exact length (70); a far-off length or junk words drop it below. */
|
||||
/**
|
||||
* Auto-confirm: title match plus artist (80) or exact length (70); a far-off length or junk words drop it below,
|
||||
* and a length more than MAX_LENGTH_DIFF seconds off never confirms.
|
||||
*/
|
||||
export function isConfident(s: Scored): boolean {
|
||||
return s.titleOk && s.score >= 70;
|
||||
return s.titleOk && s.score >= 70 && (s.lengthDiff === null || s.lengthDiff <= MAX_LENGTH_DIFF);
|
||||
}
|
||||
|
||||
/** Why a result was not confirmed, in plain words for the admin list ("가수 다름: Ashiyane"). Empty when confident. */
|
||||
export function notConfidentReasons(s: Scored): string[] {
|
||||
if (isConfident(s)) return [];
|
||||
const out: string[] = [];
|
||||
if (!s.titleOk) out.push(`제목 다름: ${s.title}`);
|
||||
if (!s.artistOk) out.push(`가수 다름: ${s.artists || '알 수 없음'}`);
|
||||
if (s.lengthDiff !== null && s.lengthDiff > MAX_LENGTH_DIFF) out.push(`길이 ${s.lengthDiff}초 차이`);
|
||||
if (s.junk) out.push('커버·노래방·라이브 등 다른 버전');
|
||||
if (s.topic === false) out.push('공식 음원 채널 아님');
|
||||
if (!out.length && s.lengthDiff === null) out.push('곡 길이 정보 없음');
|
||||
if (!out.length) out.push('점수 부족');
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
/** Song catalog queries for the admin pages (docs/02 노래 관리 화면). */
|
||||
import type { Database } from 'bun:sqlite';
|
||||
import { explainCandidate } from './youtube-review';
|
||||
import { chosung, songMatchKey } from '@mq/shared';
|
||||
|
||||
export function catalogStats(db: Database) {
|
||||
@@ -126,8 +127,11 @@ export function getSong(db: Database, id: number) {
|
||||
lyrics: db.query<{ text: string; source: string }, [number]>('SELECT text, source FROM lyrics WHERE song_id = ?').get(id),
|
||||
sources: db.query<{ source: string; ref: string }, [number]>('SELECT source, ref FROM song_sources WHERE song_id = ?').all(id),
|
||||
candidates: db
|
||||
.query<Record<string, unknown>, [number]>('SELECT video_id, title, channel, duration_sec, score, source FROM youtube_candidates WHERE song_id = ? ORDER BY score DESC')
|
||||
.all(id),
|
||||
.query<{ video_id: string; title: string | null; channel: string | null; duration_sec: number | null; score: number; source: string }, [number]>(
|
||||
'SELECT video_id, title, channel, duration_sec, score, source FROM youtube_candidates WHERE song_id = ? ORDER BY score DESC',
|
||||
)
|
||||
.all(id)
|
||||
.map((c) => ({ ...c, reasons: explainCandidate(db, song as { title: string; artist: string; duration_sec: number | null }, c) })),
|
||||
signals: db.query<{ kind: string; value: number }, [number]>('SELECT kind, value FROM signals WHERE song_id = ?').all(id),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -3,6 +3,31 @@
|
||||
* Tabs split "searched but nothing found" from "not searched yet" (both are youtube_status 'none').
|
||||
*/
|
||||
import type { Database } from 'bun:sqlite';
|
||||
import { learnArtistAliases } from '../collector/artist-aliases';
|
||||
import { notConfidentReasons, scoreResult, type ArtistAliases } from '../collector/score';
|
||||
|
||||
let aliasCache: { db: Database; at: number; map: ArtistAliases } | null = null;
|
||||
/** Learnt artist names change only when songs get confirmed: re-learn at most every 10 minutes. */
|
||||
function aliasesOf(db: Database, now = Date.now()): ArtistAliases {
|
||||
if (!aliasCache || aliasCache.db !== db || now - aliasCache.at > 600_000) aliasCache = { db, at: now, map: learnArtistAliases(db) };
|
||||
return aliasCache.map;
|
||||
}
|
||||
|
||||
type CandInfo = { video_id: string; title: string | null; channel: string | null; duration_sec: number | null; source: string };
|
||||
/**
|
||||
* Why the matcher would not confirm this result, in plain words ("가수 다름: Ashiyane", "길이 40초 차이");
|
||||
* empty when it would. norabose videos carry no title/artist/length, so they are only labelled as such.
|
||||
*/
|
||||
export function explainCandidate(db: Database, song: { title: string; artist: string; duration_sec: number | null }, c: CandInfo): string[] {
|
||||
if (c.source === 'norabose' && !c.title) return ['노라보세 영상이라 제목·가수·길이 정보 없음'];
|
||||
const scored = scoreResult(
|
||||
{ title: song.title, artist: song.artist, durationSec: song.duration_sec },
|
||||
{ videoId: c.video_id, title: c.title ?? '', artists: c.channel ?? '', album: null, durationSec: c.duration_sec },
|
||||
-1,
|
||||
aliasesOf(db),
|
||||
);
|
||||
return notConfidentReasons(scored);
|
||||
}
|
||||
|
||||
export const REVIEW_TABS = ['matched', 'candidate', 'missing', 'noorig', 'dead', 'pending'] as const;
|
||||
export type ReviewTab = (typeof REVIEW_TABS)[number];
|
||||
@@ -138,7 +163,11 @@ export interface ReviewRow {
|
||||
score: number | null;
|
||||
note: string | null;
|
||||
touched: number;
|
||||
candidates: { video_id: string; title: string | null; channel: string | null; duration_sec: number | null; score: number; source: string }[];
|
||||
/** The real artist of the attached video (YouTube Music's artist field), not the channel name ("Release - Topic"). */
|
||||
video_artist: string | null;
|
||||
/** 후보 only: why the attached video was not confirmed automatically. */
|
||||
reasons: string[];
|
||||
candidates: { video_id: string; title: string | null; channel: string | null; duration_sec: number | null; score: number; source: string; reasons: string[] }[];
|
||||
}
|
||||
|
||||
export function reviewList(db: Database, tab: ReviewTab, opts: { q?: string; page?: number } = {}) {
|
||||
@@ -155,7 +184,7 @@ export function reviewList(db: Database, tab: ReviewTab, opts: { q?: string; pag
|
||||
// 대기: in the order the matcher will take them (most popular level first); the rest: latest first.
|
||||
const order = tab === 'pending' ? 'lv.level IS NULL, lv.level, s.id' : tab === 'dead' ? 's.updated_at DESC, s.id DESC' : 'touched DESC, s.id DESC';
|
||||
const rows = db
|
||||
.query<Omit<ReviewRow, 'candidates'>, string[]>(
|
||||
.query<Omit<ReviewRow, 'candidates' | 'video_artist' | 'reasons'>, string[]>(
|
||||
`SELECT s.id, s.title, s.artist, s.year, lv.level, s.duration_sec, s.youtube_id, s.youtube_status, s.youtube_source, s.youtube_channel, s.youtube_title,
|
||||
(SELECT c.score FROM youtube_candidates c WHERE c.song_id = s.id AND c.video_id = s.youtube_id) score, s.note,
|
||||
${tab === 'dead' ? 's.updated_at' : TOUCHED} touched
|
||||
@@ -163,20 +192,37 @@ export function reviewList(db: Database, tab: ReviewTab, opts: { q?: string; pag
|
||||
WHERE ${w} ORDER BY ${order} LIMIT ${REVIEW_PAGE} OFFSET ${page * REVIEW_PAGE}`,
|
||||
)
|
||||
.all(...args);
|
||||
const cands = new Map<number, ReviewRow['candidates']>();
|
||||
type C = Omit<ReviewRow['candidates'][number], 'reasons'>;
|
||||
const all = new Map<number, C[]>();
|
||||
if (rows.length) {
|
||||
for (const c of db
|
||||
.query<ReviewRow['candidates'][number] & { song_id: number }, number[]>(
|
||||
.query<C & { song_id: number }, number[]>(
|
||||
`SELECT song_id, video_id, title, channel, duration_sec, score, source FROM youtube_candidates
|
||||
WHERE song_id IN (${rows.map(() => '?').join(',')}) ORDER BY score DESC`,
|
||||
)
|
||||
.all(...rows.map((r) => r.id))) {
|
||||
const list = cands.get(c.song_id) ?? [];
|
||||
if (list.length < 6) list.push({ video_id: c.video_id, title: c.title, channel: c.channel, duration_sec: c.duration_sec, score: c.score, source: c.source });
|
||||
cands.set(c.song_id, list);
|
||||
const list = all.get(c.song_id) ?? [];
|
||||
list.push({ video_id: c.video_id, title: c.title, channel: c.channel, duration_sec: c.duration_sec, score: c.score, source: c.source });
|
||||
all.set(c.song_id, list);
|
||||
}
|
||||
}
|
||||
return { total, page, pageSize: REVIEW_PAGE, rows: rows.map((r) => ({ ...r, candidates: cands.get(r.id) ?? [] })) };
|
||||
return {
|
||||
total,
|
||||
page,
|
||||
pageSize: REVIEW_PAGE,
|
||||
rows: rows.map((r): ReviewRow => {
|
||||
const list = all.get(r.id) ?? [];
|
||||
// the attached video's own row: prefer YouTube Music's (it knows the artist), else norabose's
|
||||
const own = list.filter((c) => c.video_id === r.youtube_id).sort((a, b) => Number(b.source === 'ytmusic') - Number(a.source === 'ytmusic'))[0];
|
||||
const explain = (c: C) => (r.youtube_status === 'candidate' ? explainCandidate(db, r, c) : []);
|
||||
return {
|
||||
...r,
|
||||
video_artist: own?.source === 'ytmusic' ? own.channel : null,
|
||||
reasons: own ? explain(own) : r.youtube_status === 'candidate' && r.youtube_id ? ['찾기 결과에 없는 영상이라 정보 없음'] : [],
|
||||
candidates: list.slice(0, 6).map((c) => ({ ...c, reasons: explain(c) })),
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -41,7 +41,7 @@ interface Detail {
|
||||
aliases: { part: 'title' | 'artist'; alias: string }[];
|
||||
lyrics: { text: string } | null;
|
||||
sources: { source: string; ref: string }[];
|
||||
candidates: { video_id: string; title: string | null; channel: string | null; duration_sec: number | null; score: number; source: string }[];
|
||||
candidates: { video_id: string; title: string | null; channel: string | null; duration_sec: number | null; score: number; source: string; reasons?: string[] }[];
|
||||
}
|
||||
interface Stats {
|
||||
total: number;
|
||||
@@ -790,6 +790,7 @@ export function SongEditor({ id, genres, onClose, onSaved }: { id: number; genre
|
||||
<span className="grow" style={{ textAlign: 'left' }}>
|
||||
{c.title ?? c.video_id}
|
||||
<span className="muted small"> · {c.channel ?? (c.source === 'norabose' ? '노라보세 영상' : '')}</span>
|
||||
{!!c.reasons?.length && <span className="warn-text small"> — {c.reasons.join(' · ')}</span>}
|
||||
</span>
|
||||
<span className="mono small">{mmss(c.duration_sec)}</span>
|
||||
</button>
|
||||
|
||||
@@ -18,6 +18,8 @@ interface Cand {
|
||||
duration_sec: number | null;
|
||||
score: number;
|
||||
source: string;
|
||||
/** Why it was not confirmed automatically ("가수 다름: Ashiyane"); empty when it would be. */
|
||||
reasons: string[];
|
||||
}
|
||||
interface Row {
|
||||
id: number;
|
||||
@@ -33,6 +35,8 @@ interface Row {
|
||||
score: number | null;
|
||||
note: string | null;
|
||||
touched: number;
|
||||
video_artist: string | null;
|
||||
reasons: string[];
|
||||
candidates: Cand[];
|
||||
}
|
||||
interface JobBase {
|
||||
@@ -418,7 +422,8 @@ export function YoutubePage() {
|
||||
{r.youtube_id ? (
|
||||
<>
|
||||
{r.youtube_title ?? r.youtube_id}
|
||||
<div className="muted">{r.youtube_channel ?? ''}</div>
|
||||
<div className="muted">{r.video_artist ? `가수: ${r.video_artist}` : (r.youtube_channel ?? '')}</div>
|
||||
{r.reasons.length > 0 && <div className="warn-text">{r.reasons.join(' · ')}</div>}
|
||||
</>
|
||||
) : r.candidates.length ? (
|
||||
<span className="muted">후보 {r.candidates.length}개 (붙은 영상 없음)</span>
|
||||
@@ -576,6 +581,7 @@ function ReviewPanel({ id, onClose, onEdit, onChanged }: { id: number; onClose:
|
||||
<span className="grow">
|
||||
{c.title ?? c.video_id}
|
||||
<span className="muted small"> · {c.channel ?? (c.source === 'norabose' ? '노라보세 영상' : '')} · {mmss(c.duration_sec)}</span>
|
||||
{c.reasons?.length > 0 && <span className="warn-text small"> — {c.reasons.join(' · ')}</span>}
|
||||
</span>
|
||||
<button type="button" className="btn btn-sm btn-ghost" aria-label={`${c.title ?? c.video_id} 듣기`} onClick={() => setListen(c.video_id)}>
|
||||
▶
|
||||
|
||||
@@ -149,6 +149,9 @@ textarea.input { padding: 10px 14px; min-height: 90px; resize: vertical; }
|
||||
@media (prefers-color-scheme: dark) { :root[data-theme='auto'] .badge.ok, :root[data-theme='auto'] .badge.teal { color: #9fd9de; } }
|
||||
.badge.warn { background: var(--warning-soft); color: #8a5a00; }
|
||||
:root[data-theme='dark'] .badge.warn { color: var(--mustard); }
|
||||
.warn-text { color: #8a5a00; font-size: 0.85em; }
|
||||
:root[data-theme='dark'] .warn-text { color: var(--mustard); }
|
||||
@media (prefers-color-scheme: dark) { :root[data-theme='auto'] .warn-text { color: var(--mustard); } }
|
||||
.chip { min-height: 30px; padding: 0 11px; border-radius: 999px; border: 1.5px solid var(--line); background: var(--surface); font: inherit; font-weight: 650; font-size: 0.87rem; display: inline-flex; align-items: center; gap: 5px; color: var(--text); cursor: pointer; }
|
||||
.chip[aria-pressed='true'] { border: 2px solid var(--ink); background: var(--primary); color: var(--on-primary); }
|
||||
|
||||
|
||||
@@ -25,3 +25,11 @@ test('song match key is stable across spellings', () => {
|
||||
expect(songMatchKey('Last Scene (feat. 원슈타인)', '아이유')).toBe('lastscene|아이유');
|
||||
expect(songMatchKey('0+0', '한로로')).toBe('00|한로로');
|
||||
});
|
||||
|
||||
test('compare keys fold Latin accents and split 및 / with', () => {
|
||||
expect(keysOverlap(artistKeys('JAY-Z'), artistKeys('JAŸ-Z'))).toBe(true);
|
||||
expect(keysOverlap(artistKeys('Beyonce'), artistKeys('Beyoncé'))).toBe(true);
|
||||
expect(artistKeys('X JAPAN 및 YOSHIKI')).toEqual(expect.arrayContaining(['xjapan', 'yoshiki']));
|
||||
expect(keysOverlap(artistKeys('Human League'), artistKeys('The Human League'))).toBe(true);
|
||||
expect(titleKeys('が')).not.toContain('か'); // kana marks stay
|
||||
});
|
||||
|
||||
@@ -12,11 +12,19 @@ export function looseKey(s: string): string {
|
||||
.replace(/[^\p{L}\p{N}]+/gu, '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Comparison key for YouTube matching only (not stored): like looseKey, but Latin letters lose their accents so
|
||||
* "JAŸ-Z" and "JAY-Z" or "Beyoncé" and "Beyonce" compare equal. Kana/Hangul marks are left alone.
|
||||
*/
|
||||
export function compareKey(s: string): string {
|
||||
return looseKey(s.normalize('NFD').replace(/(\p{Script=Latin})\p{M}+/gu, '$1').normalize('NFC'));
|
||||
}
|
||||
|
||||
/** Title variants: as is, without feat/prod tails, without any parenthesised part, and each parenthesised part alone. */
|
||||
export function titleKeys(title: string): string[] {
|
||||
const out = new Set<string>();
|
||||
const add = (s: string) => {
|
||||
const k = looseKey(s);
|
||||
const k = compareKey(s);
|
||||
if (k) out.add(k);
|
||||
};
|
||||
add(title);
|
||||
@@ -27,15 +35,17 @@ export function titleKeys(title: string): string[] {
|
||||
return [...out];
|
||||
}
|
||||
|
||||
/** Artist variants: split on , & / x and "(…)" so "RESCENE (리센느)" matches both names. */
|
||||
/** Artist variants: split on , & / x 및 with feat and "(…)" so "RESCENE (리센느)" matches both names. */
|
||||
export function artistKeys(artist: string): string[] {
|
||||
const out = new Set<string>();
|
||||
const parts = artist.split(/\s*(?:,|&|\/|\bx\b|\band\b|·|feat\.?)\s*/i);
|
||||
const parts = artist.split(/\s*(?:,|&|\/|×|\sx\s|\band\b|\s및\s|\bwith\b|\bvs\.?(?=\s)|·|\bfeat\.?|\bft\.)\s*/i);
|
||||
for (const p of [artist, ...parts]) {
|
||||
const k = looseKey(p);
|
||||
const k = compareKey(p);
|
||||
if (k) out.add(k);
|
||||
const noThe = compareKey(p.replace(/^\s*the\s+/i, ''));
|
||||
if (noThe) out.add(noThe);
|
||||
const inner = /^(.*?)\s*\(([^)]+)\)\s*$/.exec(p);
|
||||
if (inner) for (const q of [inner[1]!, inner[2]!]) if (looseKey(q)) out.add(looseKey(q));
|
||||
if (inner) for (const q of [inner[1]!, inner[2]!]) if (compareKey(q)) out.add(compareKey(q));
|
||||
}
|
||||
return [...out];
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user