유튜브 원곡 찾기: 인기곡(전체 난이도 낮은 곡)부터, --deadline(epoch ms)이 지나면 깔끔히 멈춤(작업은 완료로 기록) — 새벽 정기 작업 전까지 이어 돌리기용

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
EJClaw
2026-10-18 23:39:22 +09:00
parent a1507e5b95
commit 453405ec81
3 changed files with 33 additions and 5 deletions

View File

@@ -2,7 +2,7 @@
* Song catalog collector (docs/07 처리 순서). Runs next to the app on the same DB (WAL):
* bun apps/server/src/collector/cli.ts import-norabose [--limit N]
* bun apps/server/src/collector/cli.ts import-sta
* bun apps/server/src/collector/cli.ts match-youtube [--limit N] [--delay ms] [--require-cookie] (default pace: 2s apart; cron caps a day at 3000)
* bun apps/server/src/collector/cli.ts match-youtube [--limit N] [--delay ms] [--deadline epochMs] [--require-cookie] (default pace: 2s apart; cron caps a day at 3000; most popular level first)
* bun apps/server/src/collector/cli.ts charts [--from YYYY] [--to YYYY] [--sources melon-ye,genie-ye,tj-ye,melon-rt,bugs-rt] [--refetch]
* bun apps/server/src/collector/cli.ts melon-detail [--limit N]
* bun apps/server/src/collector/cli.ts heatmap [--limit N]
@@ -48,7 +48,7 @@ switch (cmd) {
case 'match-youtube': {
const searcher = await pickSearcher(process.env.YT_COOKIE_PATH ?? '/home/claude/.config/music_quiz/ytm_cookie.txt', console.log, rest.includes('--require-cookie'));
console.log(`search: ${searcher.kind}`);
console.log(await matchYoutube(db, searcher, { limit: flag('limit'), delayMs: flag('delay'), source: searcher.kind }));
console.log(await matchYoutube(db, searcher, { limit: flag('limit'), delayMs: flag('delay'), deadline: flag('deadline'), source: searcher.kind }));
break;
}
case 'charts': {

View File

@@ -111,3 +111,22 @@ test('YouTube matching: confident → matched, unsure → candidate fallback, no
expect(await matchYoutube(db, { searchSongs: async () => { throw new Error('should not search'); } }, { delayMs: 0 })).toEqual({ matched: 0, candidate: 0, none: 0 });
expect(listSongs(db, { youtube: 'matched' }).total).toBe(1);
});
test('YouTube matching asks the most popular level first and stops at the deadline', async () => {
const db = freshDb();
const c = new Catalog(db);
const [rare] = c.upsert({ title: '숨은곡', artists: [{ name: '가' }], source: 't', ref: '1' });
const [hit] = c.upsert({ title: '대표곡', artists: [{ name: '나' }], source: 't', ref: '2' });
const [mid] = c.upsert({ title: '중간곡', artists: [{ name: '다' }], source: 't', ref: '3' });
c.autoLevel(rare!, 'all', 9);
c.autoLevel(hit!, 'all', 1);
c.autoLevel(mid!, 'all', 5);
const asked: string[] = [];
const searchSongs = async (q: string) => (asked.push(q), []);
const noEmbed = async () => ({ state: 'noembed' as const });
await matchYoutube(db, { searchSongs }, { delayMs: 0, limit: 2 }, noEmbed);
expect(asked).toEqual(['나 대표곡', '다 중간곡']);
await matchYoutube(db, { searchSongs }, { delayMs: 0, deadline: Date.now() - 1 }, noEmbed);
expect(asked.length).toBe(2);
expect(db.query<{ status: string }, []>("SELECT status FROM collect_jobs WHERE kind = 'match-youtube' ORDER BY id DESC LIMIT 1").get()!.status).toBe('done');
});

View File

@@ -20,6 +20,8 @@ export interface MatchOptions {
limit?: number;
delayMs?: number;
retryDays?: number;
/** Stop cleanly (job done) once this time (epoch ms) has passed, e.g. before the nightly cron. */
deadline?: number;
/** Where results come from: recorded on songs and candidates. */
source?: 'ytmusic' | 'youtube';
log?: (s: string) => void;
@@ -33,13 +35,15 @@ export async function matchYoutube(db: Database, ytm: Pick<YtMusic, 'searchSongs
.query<Row, [string, number, number]>(
`SELECT s.id, s.title, s.artist, s.duration_sec FROM songs s
LEFT JOIN signals t ON t.song_id = s.id AND t.kind = ?
LEFT JOIN song_levels l ON l.song_id = s.id AND l.scope = 'all'
WHERE s.deleted_at IS NULL AND s.youtube_status IN ('none','candidate') AND COALESCE(s.youtube_source, '') <> 'admin'
AND (t.value IS NULL OR t.value < ?)
ORDER BY s.id LIMIT ?`,
ORDER BY l.level IS NULL, l.level, s.id LIMIT ?`,
)
.all(triedKind, retryBefore, opts.limit ?? 1_000_000);
const job = new Job(db, 'match-youtube', rows.length);
const stats = { matched: 0, candidate: 0, none: 0 };
let tried = 0;
const src = opts.source ?? 'ytmusic';
/**
* Writes the result only if the song is still open for automatic matching: an admin may have saved a video
@@ -57,6 +61,10 @@ export async function matchYoutube(db: Database, ytm: Pick<YtMusic, 'searchSongs
db.query<{ n: number }, [number]>("SELECT COUNT(*) n FROM songs WHERE id = ? AND COALESCE(youtube_source, '') = 'admin'").get(id)!.n > 0;
try {
for (const [i, s] of rows.entries()) {
if (opts.deadline !== undefined && Date.now() >= opts.deadline) {
job.log(`deadline reached after ${i}/${rows.length}`);
break;
}
let results: YtmSong[];
const query = `${s.artist.split(', ')[0]} ${s.title}`;
try {
@@ -105,11 +113,12 @@ export async function matchYoutube(db: Database, ytm: Pick<YtMusic, 'searchSongs
if (!done) stats.none++;
}
db.query('INSERT INTO signals (song_id, kind, value, fetched_at) VALUES (?, ?, ?, ?) ON CONFLICT(song_id, kind) DO UPDATE SET value = excluded.value, fetched_at = excluded.fetched_at').run(s.id, triedKind, Date.now(), Date.now());
job.progress(i + 1);
tried = i + 1;
job.progress(tried);
if ((i + 1) % 50 === 0) job.log(`youtube ${i + 1}/${rows.length} matched ${stats.matched} candidate ${stats.candidate} none ${stats.none}`);
await sleep(opts.delayMs ?? 2000);
}
job.log(`youtube done ${rows.length}: matched ${stats.matched} candidate ${stats.candidate} none ${stats.none}`);
job.log(`youtube done ${tried}/${rows.length}: matched ${stats.matched} candidate ${stats.candidate} none ${stats.none}`);
job.finish('done');
} catch (e) {
job.log(`stopped: ${(e as Error).message}`);