유튜브 원곡 찾기: 인기곡(전체 난이도 낮은 곡)부터, --deadline(epoch ms)이 지나면 깔끔히 멈춤(작업은 완료로 기록) — 새벽 정기 작업 전까지 이어 돌리기용
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -2,7 +2,7 @@
|
||||
* Song catalog collector (docs/07 처리 순서). Runs next to the app on the same DB (WAL):
|
||||
* bun apps/server/src/collector/cli.ts import-norabose [--limit N]
|
||||
* bun apps/server/src/collector/cli.ts import-sta
|
||||
* bun apps/server/src/collector/cli.ts match-youtube [--limit N] [--delay ms] [--require-cookie] (default pace: 2s apart; cron caps a day at 3000)
|
||||
* bun apps/server/src/collector/cli.ts match-youtube [--limit N] [--delay ms] [--deadline epochMs] [--require-cookie] (default pace: 2s apart; cron caps a day at 3000; most popular level first)
|
||||
* bun apps/server/src/collector/cli.ts charts [--from YYYY] [--to YYYY] [--sources melon-ye,genie-ye,tj-ye,melon-rt,bugs-rt] [--refetch]
|
||||
* bun apps/server/src/collector/cli.ts melon-detail [--limit N]
|
||||
* bun apps/server/src/collector/cli.ts heatmap [--limit N]
|
||||
@@ -48,7 +48,7 @@ switch (cmd) {
|
||||
case 'match-youtube': {
|
||||
const searcher = await pickSearcher(process.env.YT_COOKIE_PATH ?? '/home/claude/.config/music_quiz/ytm_cookie.txt', console.log, rest.includes('--require-cookie'));
|
||||
console.log(`search: ${searcher.kind}`);
|
||||
console.log(await matchYoutube(db, searcher, { limit: flag('limit'), delayMs: flag('delay'), source: searcher.kind }));
|
||||
console.log(await matchYoutube(db, searcher, { limit: flag('limit'), delayMs: flag('delay'), deadline: flag('deadline'), source: searcher.kind }));
|
||||
break;
|
||||
}
|
||||
case 'charts': {
|
||||
|
||||
@@ -111,3 +111,22 @@ test('YouTube matching: confident → matched, unsure → candidate fallback, no
|
||||
expect(await matchYoutube(db, { searchSongs: async () => { throw new Error('should not search'); } }, { delayMs: 0 })).toEqual({ matched: 0, candidate: 0, none: 0 });
|
||||
expect(listSongs(db, { youtube: 'matched' }).total).toBe(1);
|
||||
});
|
||||
|
||||
test('YouTube matching asks the most popular level first and stops at the deadline', async () => {
|
||||
const db = freshDb();
|
||||
const c = new Catalog(db);
|
||||
const [rare] = c.upsert({ title: '숨은곡', artists: [{ name: '가' }], source: 't', ref: '1' });
|
||||
const [hit] = c.upsert({ title: '대표곡', artists: [{ name: '나' }], source: 't', ref: '2' });
|
||||
const [mid] = c.upsert({ title: '중간곡', artists: [{ name: '다' }], source: 't', ref: '3' });
|
||||
c.autoLevel(rare!, 'all', 9);
|
||||
c.autoLevel(hit!, 'all', 1);
|
||||
c.autoLevel(mid!, 'all', 5);
|
||||
const asked: string[] = [];
|
||||
const searchSongs = async (q: string) => (asked.push(q), []);
|
||||
const noEmbed = async () => ({ state: 'noembed' as const });
|
||||
await matchYoutube(db, { searchSongs }, { delayMs: 0, limit: 2 }, noEmbed);
|
||||
expect(asked).toEqual(['나 대표곡', '다 중간곡']);
|
||||
await matchYoutube(db, { searchSongs }, { delayMs: 0, deadline: Date.now() - 1 }, noEmbed);
|
||||
expect(asked.length).toBe(2);
|
||||
expect(db.query<{ status: string }, []>("SELECT status FROM collect_jobs WHERE kind = 'match-youtube' ORDER BY id DESC LIMIT 1").get()!.status).toBe('done');
|
||||
});
|
||||
|
||||
@@ -20,6 +20,8 @@ export interface MatchOptions {
|
||||
limit?: number;
|
||||
delayMs?: number;
|
||||
retryDays?: number;
|
||||
/** Stop cleanly (job done) once this time (epoch ms) has passed, e.g. before the nightly cron. */
|
||||
deadline?: number;
|
||||
/** Where results come from: recorded on songs and candidates. */
|
||||
source?: 'ytmusic' | 'youtube';
|
||||
log?: (s: string) => void;
|
||||
@@ -33,13 +35,15 @@ export async function matchYoutube(db: Database, ytm: Pick<YtMusic, 'searchSongs
|
||||
.query<Row, [string, number, number]>(
|
||||
`SELECT s.id, s.title, s.artist, s.duration_sec FROM songs s
|
||||
LEFT JOIN signals t ON t.song_id = s.id AND t.kind = ?
|
||||
LEFT JOIN song_levels l ON l.song_id = s.id AND l.scope = 'all'
|
||||
WHERE s.deleted_at IS NULL AND s.youtube_status IN ('none','candidate') AND COALESCE(s.youtube_source, '') <> 'admin'
|
||||
AND (t.value IS NULL OR t.value < ?)
|
||||
ORDER BY s.id LIMIT ?`,
|
||||
ORDER BY l.level IS NULL, l.level, s.id LIMIT ?`,
|
||||
)
|
||||
.all(triedKind, retryBefore, opts.limit ?? 1_000_000);
|
||||
const job = new Job(db, 'match-youtube', rows.length);
|
||||
const stats = { matched: 0, candidate: 0, none: 0 };
|
||||
let tried = 0;
|
||||
const src = opts.source ?? 'ytmusic';
|
||||
/**
|
||||
* Writes the result only if the song is still open for automatic matching: an admin may have saved a video
|
||||
@@ -57,6 +61,10 @@ export async function matchYoutube(db: Database, ytm: Pick<YtMusic, 'searchSongs
|
||||
db.query<{ n: number }, [number]>("SELECT COUNT(*) n FROM songs WHERE id = ? AND COALESCE(youtube_source, '') = 'admin'").get(id)!.n > 0;
|
||||
try {
|
||||
for (const [i, s] of rows.entries()) {
|
||||
if (opts.deadline !== undefined && Date.now() >= opts.deadline) {
|
||||
job.log(`deadline reached after ${i}/${rows.length}`);
|
||||
break;
|
||||
}
|
||||
let results: YtmSong[];
|
||||
const query = `${s.artist.split(', ')[0]} ${s.title}`;
|
||||
try {
|
||||
@@ -105,11 +113,12 @@ export async function matchYoutube(db: Database, ytm: Pick<YtMusic, 'searchSongs
|
||||
if (!done) stats.none++;
|
||||
}
|
||||
db.query('INSERT INTO signals (song_id, kind, value, fetched_at) VALUES (?, ?, ?, ?) ON CONFLICT(song_id, kind) DO UPDATE SET value = excluded.value, fetched_at = excluded.fetched_at').run(s.id, triedKind, Date.now(), Date.now());
|
||||
job.progress(i + 1);
|
||||
tried = i + 1;
|
||||
job.progress(tried);
|
||||
if ((i + 1) % 50 === 0) job.log(`youtube ${i + 1}/${rows.length} matched ${stats.matched} candidate ${stats.candidate} none ${stats.none}`);
|
||||
await sleep(opts.delayMs ?? 2000);
|
||||
}
|
||||
job.log(`youtube done ${rows.length}: matched ${stats.matched} candidate ${stats.candidate} none ${stats.none}`);
|
||||
job.log(`youtube done ${tried}/${rows.length}: matched ${stats.matched} candidate ${stats.candidate} none ${stats.none}`);
|
||||
job.finish('done');
|
||||
} catch (e) {
|
||||
job.log(`stopped: ${(e as Error).message}`);
|
||||
|
||||
Reference in New Issue
Block a user