diff --git a/apps/server/src/collector/age-audio.test.ts b/apps/server/src/collector/age-audio.test.ts index 3f1e066..e0b3ab7 100644 --- a/apps/server/src/collector/age-audio.test.ts +++ b/apps/server/src/collector/age-audio.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from 'bun:test'; -import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { DEFAULT_ROOM_CONFIG } from '@mq/shared'; import { openDb } from '../db'; @@ -9,6 +10,7 @@ import { loudnessGain, quizSong } from '../songs/quiz'; import { tempDbPath } from '../test-utils'; import { cookieForms, collectAgeAudio } from './age-audio'; import { Catalog } from './catalog'; +import { cooldownUntil } from './cookie-jar'; import { Blocked } from './charts'; import { collectLoudness, isAgeGated } from './loudness'; @@ -120,4 +122,20 @@ describe('연령제한 곡', () => { expect(ids.every((id) => !quizSong(db, id)!.audio)).toBe(true); expect(existsSync(dirname(audioFile(dir, 'x')))).toBe(true); }); + + test('a block while downloading starts the 429 pause; during it the downloads wait (nothing is tried)', async () => { + const { db, dir } = setup(1); + db.query('UPDATE songs SET age_video = youtube_id').run(); + const cookiePath = join(mkdtempSync(join(tmpdir(), 'mq-age-')), 'ytm_cookie.txt'); + writeFileSync(cookiePath, 'SAPISID=a; __Secure-1PSID=b'); + let downloads = 0; + const blocked = { delayMs: 0, cookiePath, fetchSignedIn: async () => ok(-9), download: async () => (downloads++, Promise.reject(new Blocked('HTTP Error 429'))) }; + await collectAgeAudio(db, dir, blocked); + expect(downloads).toBe(1); + expect(cooldownUntil(cookiePath)).toBeGreaterThan(Date.now() + 2.9 * 3600_000); + db.query('DELETE FROM signals').run(); + await collectAgeAudio(db, dir, blocked); + expect(downloads).toBe(1); + expect(db.query<{ status: string; log: string }, []>("SELECT status, log FROM collect_jobs WHERE kind = 'age-audio' ORDER BY id DESC").get()).toMatchObject({ status: 'failed', log: expect.stringContaining('429') }); + }); }); diff --git a/apps/server/src/collector/age-audio.ts b/apps/server/src/collector/age-audio.ts index 435a1c6..57944b0 100644 --- a/apps/server/src/collector/age-audio.ts +++ b/apps/server/src/collector/age-audio.ts @@ -15,7 +15,7 @@ import { join } from 'node:path'; import { audioFile } from '../songs/audio'; import { Catalog, Job, sleep } from './catalog'; import { Blocked } from './charts'; -import { cookieForms, jarChanges, mergeCookies, writeCookieFile } from './cookie-jar'; +import { cookieForms, cooldownUntil, jarChanges, mergeCookies, RateLimited, startCooldown, writeCookieFile } from './cookie-jar'; import { parseHeatmap, pickHighlight } from './heatmap'; import { fetchWatchPage, isAgeGated, parseLoudness, saveLoudness } from './loudness'; @@ -124,14 +124,16 @@ export async function collectAgeAudio(db: Database, dir: string, opts: AgeAudioO await sleep(opts.delayMs ?? 3000); } - // 2. download with the adult-verified account + // 2. download with the adult-verified account (not during a 429 pause; a block here starts one — cookie-jar.ts) + // (tests that inject both without a cookie path touch no cookie file) + const path = opts.fetchSignedIn && opts.download ? opts.cookiePath : (opts.cookiePath ?? process.env.YT_COOKIE_PATH ?? '/home/claude/.config/music_quiz/ytm_cookie.txt'); + if (path && cooldownUntil(path) > Date.now()) throw new RateLimited(cooldownUntil(path)); const todo = pending(); job.progress(done, recheck.length + todo.length); let fetchSignedIn = opts.fetchSignedIn; let download = opts.download; if (todo.length && (!fetchSignedIn || !download)) { - const path = opts.cookiePath ?? process.env.YT_COOKIE_PATH ?? '/home/claude/.config/music_quiz/ytm_cookie.txt'; - if (!existsSync(path)) throw new CookieProblem(`쿠키 파일이 없어요: ${path}`); + if (!path || !existsSync(path)) throw new CookieProblem(`쿠키 파일이 없어요: ${path}`); tmp = mkdtempSync(join(tmpdir(), 'mq-cookie-')); const cookieFile = join(tmp, 'cookies.txt'); // the file is read for every song: the server keeps rotating its values (cookie-keeper.ts) @@ -152,6 +154,7 @@ export async function collectAgeAudio(db: Database, dir: string, opts: AgeAudioO stats.downloaded++; job.log(`${s.youtube_id} downloaded`); } catch (e) { + if (e instanceof Blocked && path) startCooldown(path); if (e instanceof Blocked || e instanceof CookieProblem) throw e; stats.failed++; cat.signal(s.id, 'yt_audio_tried', Date.now()); @@ -173,7 +176,7 @@ export async function collectAgeAudio(db: Database, dir: string, opts: AgeAudioO } catch (e) { job.log(`stopped: ${(e as Error).message}`); job.finish('failed'); - if (!(e instanceof Blocked || e instanceof CookieProblem)) throw e; + if (!(e instanceof Blocked || e instanceof CookieProblem || e instanceof RateLimited)) throw e; } finally { if (tmp) rmSync(tmp, { recursive: true, force: true }); } diff --git a/apps/server/src/collector/cli.ts b/apps/server/src/collector/cli.ts index 60db2c2..05af690 100644 --- a/apps/server/src/collector/cli.ts +++ b/apps/server/src/collector/cli.ts @@ -2,7 +2,7 @@ * Song catalog collector (docs/07 처리 순서). Runs next to the app on the same DB (WAL): * bun apps/server/src/collector/cli.ts import-norabose [--limit N] * bun apps/server/src/collector/cli.ts import-sta - * bun apps/server/src/collector/cli.ts match-youtube [--limit N] [--delay ms] [--deadline epochMs] [--require-cookie] (default pace: 2s apart; cron caps a day at 3000; most popular level first) + * bun apps/server/src/collector/cli.ts match-youtube [--limit N] [--delay ms] [--deadline epochMs] [--require-cookie] (default pace: 5s apart, stops at the first 429; cron caps a day at 300; most popular level first) * bun apps/server/src/collector/cli.ts reconfirm-youtube [--dry-run] [--delay ms] (후보 곡을 이미 저장된 검색 결과로 현재 기준에 다시 비교해 확정) * bun apps/server/src/collector/cli.ts research-youtube [--dry-run] [--limit N] [--delay ms] [--deadline epochMs] (가수가 다른 문자로 적혀 결과에 그 가수가 없던 후보 곡을 영어/한글 이름으로 다시 검색; 쿠키가 풀렸으면 일반 유튜브 검색) * bun apps/server/src/collector/cli.ts charts [--from YYYY] [--to YYYY] [--sources melon-ye,genie-ye,tj-ye,melon-rt,bugs-rt] [--refetch] diff --git a/apps/server/src/collector/collector.test.ts b/apps/server/src/collector/collector.test.ts index e32633f..fcdd32a 100644 --- a/apps/server/src/collector/collector.test.ts +++ b/apps/server/src/collector/collector.test.ts @@ -9,7 +9,7 @@ import { matchYoutube, reconfirmCandidates, researchYoutube } from './match-yout import { isConfident, notConfidentReasons, scoreResult } from './score'; import { soundsLike } from './sound'; import { learnArtistAliases } from './artist-aliases'; -import { parseDuration, parseSearch, type YtmSong } from './ytmusic'; +import { parseDuration, parseSearch, RateLimited, type YtmSong } from './ytmusic'; import { catalogStats, listSongs } from '../songs/store'; import { tempDbPath } from '../test-utils'; @@ -315,3 +315,20 @@ test('YouTube matching asks the most popular level first and stops at the deadli expect(asked.length).toBe(2); expect(db.query<{ status: string }, []>("SELECT status FROM collect_jobs WHERE kind = 'match-youtube' ORDER BY id DESC LIMIT 1").get()!.status).toBe('done'); }); + +test('YouTube matching stops at the first 429 (no retry a minute later) and leaves the job stopped', async () => { + const db = freshDb(); + const c = new Catalog(db); + c.upsert({ title: '곡1', artists: [{ name: '가' }], source: 't', ref: '1' }); + c.upsert({ title: '곡2', artists: [{ name: '나' }], source: 't', ref: '2' }); + let n = 0; + const searchSongs = async () => { + n++; + throw new RateLimited(Date.now() + 3 * 3600_000); + }; + const started = Date.now(); + await expect(matchYoutube(db, { searchSongs }, { delayMs: 0 })).rejects.toBeInstanceOf(RateLimited); + expect(n).toBe(1); + expect(Date.now() - started).toBeLessThan(5000); + expect(db.query<{ status: string; log: string }, []>("SELECT status, log FROM collect_jobs WHERE kind = 'match-youtube' ORDER BY id DESC LIMIT 1").get()).toMatchObject({ status: 'stopped', log: expect.stringContaining('429') }); +}); diff --git a/apps/server/src/collector/cookie-jar.ts b/apps/server/src/collector/cookie-jar.ts index ea1c497..b801de0 100644 --- a/apps/server/src/collector/cookie-jar.ts +++ b/apps/server/src/collector/cookie-jar.ts @@ -3,21 +3,73 @@ * Google keeps changing some of its values (__Secure-*PSIDTS, *SIDCC); the server writes them back here (cookie-keeper.ts), * so every reader takes the file fresh instead of keeping a copy. */ -import { writeFileSync } from 'node:fs'; +import { readFileSync, writeFileSync } from 'node:fs'; /** Returns both forms of the file: the header for fetch, the cookies.txt for yt-dlp. */ export function cookieForms(raw: string): { header: string; netscape: string } { const text = raw.trim(); if (text.includes('\t')) { const rows = netscapeRows(text).filter((p) => /youtube\.com$/.test(p[0]!)); - return { header: rows.map((p) => `${p[5]}=${p[6]!.trim()}`).join('; '), netscape: `${text}\n` }; + return { header: fillSignIn(rows.map((p) => [p[5]!, p[6]!.trim()] as [string, string])).map(([k, v]) => `${k}=${v}`).join('; '), netscape: `${text}\n` }; } const exp = Math.floor(Date.now() / 1000) + 365 * 86400; - const pairs = headerPairs(text); + const pairs = fillSignIn(headerPairs(text)); const lines = pairs.map(([k, v]) => ['.youtube.com', 'TRUE', '/', 'TRUE', exp, k, v].join('\t')); return { header: pairs.map(([k, v]) => `${k}=${v}`).join('; '), netscape: ['# Netscape HTTP Cookie File', ...lines, ''].join('\n') }; } +/** + * A cookie copied from a request another site sent to YouTube (an embedded player) has only the cross-site copies + * (__Secure-3P*) of the login values. SAPISID — the one the request signature (SAPISIDHASH) is made from — has the same + * value as __Secure-3PAPISID / __Secure-1PAPISID, so a missing one is filled from them. The file itself is left as given. + * (__Secure-1PSID and SID differ from __Secure-3PSID and cannot be filled.) + */ +function fillSignIn(pairs: [string, string][]): [string, string][] { + const has = (k: string) => pairs.some(([n]) => n === k); + const apisid = pairs.find(([k]) => k === '__Secure-3PAPISID' || k === '__Secure-1PAPISID')?.[1]; + return !has('SAPISID') && apisid ? [...pairs, ['SAPISID', apisid]] : pairs; +} + +/** Signed-in values are there: SAPISID (or a copy to fill it from) and a session id of either kind. */ +export function hasSignIn(header: string): boolean { + return /(?:^|; )SAPISID=/.test(header) && /(?:^|; )__Secure-[13]PSID=/.test(header); +} + +/** + * YouTube said 429 (too many requests) to this cookie's account / our address: everything that uses the cookie pauses + * until the time in `.cooldown` (the server's keeper and the collector CLI are separate processes). + */ +export const COOLDOWN_MS = 3 * 3600_000; +const cooldownFile = (path: string) => `${path}.cooldown`; + +export function cooldownUntil(path: string): number { + try { + return Number(readFileSync(cooldownFile(path), 'utf8').trim()) || 0; + } catch { + return 0; + } +} + +export function startCooldown(path: string, now = Date.now()): number { + const until = Math.max(cooldownUntil(path), now + COOLDOWN_MS); + try { + writeFileSync(cooldownFile(path), String(until)); + } catch { + // read-only place: the caller still stops on its own + } + return until; +} + +/** "14:05" (Korea time) for messages. */ +export const clock = (t: number) => new Date(t).toLocaleTimeString('ko-KR', { timeZone: 'Asia/Seoul', hour: '2-digit', minute: '2-digit', hour12: false }); + +/** Thrown while the 429 pause lasts (and by the request that got the 429). */ +export class RateLimited extends Error { + constructor(public until: number) { + super(`유튜브가 요청이 너무 많다고 해서(429) ${clock(until)}까지 쉬어요`); + } +} + function headerPairs(text: string): [string, string][] { return text .split(/;\s*/) diff --git a/apps/server/src/collector/cookie-keeper.test.ts b/apps/server/src/collector/cookie-keeper.test.ts index 7a3d27c..bfa8850 100644 --- a/apps/server/src/collector/cookie-keeper.test.ts +++ b/apps/server/src/collector/cookie-keeper.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from 'bun:test'; import { mkdtempSync, readFileSync, statSync, utimesSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { cookieForms, jarChanges, mergeCookies, parseSetCookies } from './cookie-jar'; +import { cookieForms, cooldownUntil, hasSignIn, jarChanges, mergeCookies, parseSetCookies, RateLimited, startCooldown } from './cookie-jar'; import { downloadWithCookie } from './age-audio'; import { CookieKeeper } from './cookie-keeper'; import { CookieError, YtMusic } from './ytmusic'; @@ -143,6 +143,98 @@ describe('CookieKeeper', () => { }); }); +describe('쿠키 일부만 복사한 경우 (다른 사이트가 보낸 요청의 쿠키: __Secure-3P* 만 있음)', () => { + const CROSS = '__Secure-3PSID=p3; __Secure-3PAPISID=ap; __Secure-3PSIDTS=ts3; __Secure-3PSIDCC=cc3; YSC=y\n'; + + test('SAPISID is filled from __Secure-3PAPISID (same value) for the header and the cookies.txt; the file is left as given', () => { + const { header, netscape } = cookieForms(CROSS); + expect(header).toBe('__Secure-3PSID=p3; __Secure-3PAPISID=ap; __Secure-3PSIDTS=ts3; __Secure-3PSIDCC=cc3; YSC=y; SAPISID=ap'); + expect(netscape).toContain('\tSAPISID\tap'); + expect(hasSignIn(header)).toBe(true); + expect(cookieForms(HEADER).header).toBe(HEADER.trim()); // a full cookie stays as it is + expect(hasSignIn(cookieForms('YSC=a; __Secure-3PAPISID=ap').header)).toBe(false); // no session id + const rotated = mergeCookies(CROSS, parseSetCookies(ROTATED, NOW)); + expect(rotated).not.toContain('SAPISID=ap;'); + expect(rotated).toContain('__Secure-3PSIDTS=new3'); + }); + + test('the keeper rotates such a cookie and YtMusic signs its requests with the filled SAPISID', async () => { + const path = tempFile(CROSS); + const sent: string[] = []; + const k = new CookieKeeper(path, { + fetch: (async (_u: string, init: RequestInit) => (sent.push((init.headers as Record).cookie!), rotateRes(200, ROTATED))) as unknown as typeof fetch, + checkLogin: async () => {}, + }); + await k.run(); + expect(k.status.state).toBe('ok'); + expect(sent[0]).toContain('SAPISID=ap'); + let auth = ''; + const ytm = new YtMusic('', (async (_u: string, init: RequestInit) => { + auth = (init.headers as Record).authorization!; + return Response.json({ responseContext: { serviceTrackingParams: [{ params: [{ key: 'logged_in', value: '1' }] }] } }); + }) as unknown as typeof fetch, path); + await ytm.searchSongs('a'); + expect(auth).toStartWith('SAPISIDHASH '); + }); +}); + +describe('429: 쿠키를 쓰는 모든 작업이 3시간 쉼', () => { + test('a search that gets 429 pauses every later request with the cookie (no request goes out)', async () => { + const path = tempFile(HEADER); + let n = 0; + const ytm = new YtMusic('', (async () => (n++, new Response('', { status: 429 }))) as unknown as typeof fetch, path); + await expect(ytm.searchSongs('a')).rejects.toBeInstanceOf(RateLimited); + expect(cooldownUntil(path)).toBeGreaterThan(Date.now() + 2.9 * 3600_000); + await expect(ytm.searchSongs('b')).rejects.toThrow('429'); + const other = new YtMusic('', (async () => (n++, Response.json({}))) as unknown as typeof fetch, path); + await expect(other.searchSongs('c')).rejects.toBeInstanceOf(RateLimited); + expect(n).toBe(1); + }); + + test('the keeper: a 429 on rotation (or on its check) waits out the pause instead of asking again in 10 minutes', async () => { + const path = tempFile(HEADER); + let status = 429; + let n = 0; + const k = new CookieKeeper(path, { fetch: (async () => (n++, rotateRes(status, ROTATED))) as unknown as typeof fetch, checkLogin: async () => {}, everyMs: 10 * 60_000 }); + await k.tick(); + expect(k.status.state).toBe('error'); + expect(k.status.message).toContain('429'); + const until = cooldownUntil(path); + expect(until).toBeGreaterThan(Date.now() + 2.9 * 3600_000); + await k.tick(Date.now() + 11 * 60_000); + expect(n).toBe(1); + status = 200; + await k.tick(until + 1); + expect(n).toBe(2); + expect(k.status.state).toBe('ok'); + + const k2 = new CookieKeeper(tempFile(HEADER), { + fetch: (async () => rotateRes(200, ROTATED)) as unknown as typeof fetch, + checkLogin: async () => { + throw new RateLimited(Date.now()); + }, + }); + await k2.run(); + expect(k2.status.state).toBe('error'); + }); + + test('a pause started by a collector also holds the keeper; a new cookie file is still checked', async () => { + const path = tempFile(HEADER); + let n = 0; + const k = new CookieKeeper(path, { fetch: (async () => (n++, rotateRes(200, ROTATED))) as unknown as typeof fetch, checkLogin: async () => {}, everyMs: 10 * 60_000 }); + await k.tick(); + expect(n).toBe(1); + startCooldown(path); + await k.tick(Date.now() + 11 * 60_000); + expect(n).toBe(1); + writeFileSync(path, HEADER.replace('p1', 'p2')); + const t = new Date(Date.now() + 5000); + utimesSync(path, t, t); + await k.tick(Date.now() + 12 * 60_000); + expect(n).toBe(2); + }); +}); + describe('YtMusic reads the cookie file again when it changes', () => { test('a long 원곡 찾기 run uses the rotated values', async () => { const path = tempFile(HEADER); diff --git a/apps/server/src/collector/cookie-keeper.ts b/apps/server/src/collector/cookie-keeper.ts index e0f0d8c..ef987eb 100644 --- a/apps/server/src/collector/cookie-keeper.ts +++ b/apps/server/src/collector/cookie-keeper.ts @@ -7,7 +7,7 @@ */ import { existsSync, readFileSync, statSync } from 'node:fs'; import type { LogFn } from '../server'; -import { cookieForms, mergeCookies, parseSetCookies, writeCookieFile } from './cookie-jar'; +import { cookieForms, cooldownUntil, hasSignIn, mergeCookies, parseSetCookies, RateLimited, startCooldown, writeCookieFile } from './cookie-jar'; import { CookieError, YtMusic } from './ytmusic'; export interface CookieStatus { @@ -70,10 +70,13 @@ export class CookieKeeper { } } - /** Runs when the file changed, or every `everyMs` while the session is alive (an ended session waits for a new file). */ + /** + * Runs when the file changed, or every `everyMs` while the session is alive (an ended session waits for a new file). + * After a 429 (here or in a collector) it waits out the pause instead of asking again in 10 minutes. + */ async tick(now = Date.now()): Promise { const changed = this.mtime() !== this.seenMtime; - const due = now - this.lastRun >= this.everyMs && this.status.state !== 'expired' && this.status.state !== 'missing'; + const due = now - this.lastRun >= this.everyMs && now >= cooldownUntil(this.path) && this.status.state !== 'expired' && this.status.state !== 'missing'; if (changed || due) await this.run(); } @@ -87,6 +90,11 @@ export class CookieKeeper { this.status = { ...this.status, state, message, checkedAt: Date.now() }; } + /** 429: rotation and every search with the cookie wait (tick skips until the pause is over). */ + private pause(): void { + this.set('error', `${new RateLimited(startCooldown(this.path)).message} — 그동안 갱신·검색을 멈춰요`); + } + private async rotate(): Promise { this.lastRun = Date.now(); this.seenMtime = this.mtime(); @@ -94,7 +102,7 @@ export class CookieKeeper { try { const raw = readFileSync(this.path, 'utf8'); const { header } = cookieForms(raw); - if (!/(?:^|; )SAPISID=/.test(header) || !/(?:^|; )__Secure-1PSID=/.test(header)) return this.set('expired', '쿠키에 로그인 값(SAPISID·__Secure-1PSID)이 없어요'); + if (!hasSignIn(header)) return this.set('expired', '쿠키에 로그인 값(SAPISID·__Secure-1PSID/3PSID)이 없어요'); const res = await this.fetch(ROTATE_URL, { method: 'POST', headers: { 'content-type': 'application/json', origin: 'https://www.youtube.com', cookie: header, 'user-agent': UA }, @@ -102,6 +110,7 @@ export class CookieKeeper { signal: AbortSignal.timeout(15_000), }); if (res.status === 401 || res.status === 403) return this.set('expired', `구글이 이 로그인 세션을 끝냈어요(갱신 ${res.status})`); + if (res.status === 429) return this.pause(); if (!res.ok) return this.set('error', `쿠키 갱신 요청 실패(${res.status}) — 다음에 다시 시도해요`); const updates = parseSetCookies(res.headers.getSetCookie()); let current = header; @@ -119,6 +128,7 @@ export class CookieKeeper { this.set('ok', null); } catch (e) { if (e instanceof CookieError) return this.set('expired', e.message); + if (e instanceof RateLimited) return this.pause(); this.set('error', `확인하지 못했어요: ${(e as Error).message} — 다음에 다시 시도해요`); } } diff --git a/apps/server/src/collector/match-youtube.ts b/apps/server/src/collector/match-youtube.ts index e441623..7a4e1e8 100644 --- a/apps/server/src/collector/match-youtube.ts +++ b/apps/server/src/collector/match-youtube.ts @@ -1,7 +1,8 @@ /** * Finds the official audio (Topic / Art Track) for each song via YouTube Music song search (docs/08 §1), * checks it can be embedded (IFrame playback, docs/12 Q1) and records candidates for the admin. - * Resumable: songs tried in the last `retryDays` are skipped. Stops on cookie problems. + * Resumable: songs tried in the last `retryDays` are skipped. Stops on cookie problems and on the first 429. + * Paced 5s apart (2s ran into a 429 and then an ended session on 11/03); the cron caps a day at 300 songs. */ import type { Database } from 'bun:sqlite'; import { Job, sleep } from './catalog'; @@ -9,7 +10,10 @@ import { checkEmbed } from './oembed'; import { addsJunk, addsVideo, confirmedDespiteLength, isConfident, namesOtherCut, scoreResult, type Scored } from './score'; import { learnArtistAliases } from './artist-aliases'; import { otherScriptNames } from './other-names'; -import { CookieError, type YtMusic, type YtmSong } from './ytmusic'; +import { CookieError, RateLimited, type YtMusic, type YtmSong } from './ytmusic'; + +/** Between YouTube Music searches. */ +export const SEARCH_DELAY_MS = 5000; interface Row { id: number; @@ -55,7 +59,8 @@ async function searchRetry(ytm: Pick, query: string, job try { return await ytm.searchSongs(query); } catch (e) { - if (e instanceof CookieError) throw e; + // 429: no retry — another try a minute later is what gets a session ended + if (e instanceof CookieError || e instanceof RateLimited) throw e; job?.log(`search failed (${(e as Error).message}), waiting 60s`); await sleep(60_000); return ytm.searchSongs(query); @@ -135,7 +140,7 @@ export async function matchYoutube(db: Database, ytm: Pick c.artistOk || c.artistSound)) { for (const name of otherNames(s.artist)) { - await sleep(opts.delayMs ?? 2000); + await sleep(opts.delayMs ?? SEARCH_DELAY_MS); // only that artist's songs count from this search: the same title by someone else is not the song const byArtist = (await searchRetry(ytm, `${name} ${s.title}`, job)).slice(0, 8).map((r, rank) => scoreResult(song, r, rank, aliases)).filter((c) => c.artistOk || c.artistSound); scored = mergeScored(scored, byArtist); @@ -168,13 +173,13 @@ export async function matchYoutube(db: Database, ytm: Pick scoreResult(song, r, rank, aliases)).filter((c) => c.artistOk || c.artistSound); stats.searches++; scored = mergeScored(scored, found); - await sleep(opts.delayMs ?? 2000); + await sleep(opts.delayMs ?? SEARCH_DELAY_MS); if (scored.some(isConfident)) break; } scored.sort((a, b) => b.score - a.score); @@ -303,7 +308,7 @@ export async function researchYoutube( job.finish('done'); } catch (e) { job.log(`stopped: ${(e as Error).message}`); - job.finish(e instanceof CookieError ? 'stopped' : 'failed'); + job.finish(e instanceof CookieError || e instanceof RateLimited ? 'stopped' : 'failed'); throw e; } return stats; diff --git a/apps/server/src/collector/ytmusic.ts b/apps/server/src/collector/ytmusic.ts index ec97936..500dbc3 100644 --- a/apps/server/src/collector/ytmusic.ts +++ b/apps/server/src/collector/ytmusic.ts @@ -3,7 +3,9 @@ * Same approach as music_bot_v2: WEB_REMIX client + SAPISIDHASH. */ import { statSync } from 'node:fs'; -import { cookieForms } from './cookie-jar'; +import { COOLDOWN_MS, cookieForms, cooldownUntil, RateLimited, startCooldown } from './cookie-jar'; + +export { RateLimited } from './cookie-jar'; export interface YtmSong { videoId: string; @@ -79,6 +81,7 @@ export class YtMusic { } private async post(endpoint: 'search' | 'next', body: Record): Promise { + if (this.path && cooldownUntil(this.path) > Date.now()) throw new RateLimited(cooldownUntil(this.path)); await this.reload(); const sapisid = /(?:^|; )SAPISID=([^;]+)/.exec(this.cookie)![1]!; const ts = Math.floor(Date.now() / 1000); @@ -98,7 +101,8 @@ export class YtMusic { signal: AbortSignal.timeout(15_000), }); if (res.status === 401 || res.status === 403) throw new CookieError(`유튜브 뮤직 인증 실패(${res.status}) — 쿠키 갱신 필요`); - if (res.status === 429) throw new Error('rate limited (429)'); + // one 429 and nothing with this cookie asks again for a while (the 11/03 burst: 429, then the session ended) + if (res.status === 429) throw new RateLimited(this.path ? startCooldown(this.path) : Date.now() + COOLDOWN_MS); if (!res.ok) throw new Error(`ytmusic ${endpoint} ${res.status}`); return res.json(); } diff --git a/apps/server/src/duel-views.test.ts b/apps/server/src/duel-views.test.ts index 792ac16..228bed2 100644 --- a/apps/server/src/duel-views.test.ts +++ b/apps/server/src/duel-views.test.ts @@ -1,7 +1,7 @@ /** 이지선다 조회수 자동: 토픽 ↔ MV pair from a YouTube Music watch response, view counts, and the upload → fetch → editor flow. */ import { afterEach, expect, test } from 'bun:test'; import { duelViewsNote, type DuelViews, type QuizItem } from '@mq/shared'; -import { CookieError, parseSongPair, type SongPair } from './collector/ytmusic'; +import { CookieError, parseSongPair, RateLimited, type SongPair } from './collector/ytmusic'; import { fetchDuelViews, youtubeViewCount, type ViewsSource } from './quizzes/duel-views'; import { TEST_SUPERADMINS, api, boot, discordLogin } from './test-utils'; @@ -213,3 +213,20 @@ test('signed out (or a song that cannot be read): songs are marked failed with t expect((await call('POST', `/quizzes/${normal}/views`, {})).res.status).toBe(400); expect((await call('GET', `/quizzes/${normal}`)).body.quiz.items[0]).toMatchObject({ views: null, viewsAuto: null }); }); + +test('429 from YouTube Music: every waiting song stops with the pause in its note (no retry)', async () => { + let n = 0; + const src = fakeSource({}, {}); + const { call, settled } = await setup(async () => ({ + ...src, + songPair: async () => { + n++; + throw new RateLimited(Date.parse('2026-11-03T15:00:00Z')); + }, + })); + const id = (await call('POST', '/quizzes/file', { name: '429', kind: 'duel', items: [row('aaaaaaaaaa1', '하나'), row('aaaaaaaaaa2', '둘')] })).body.id as number; + const items = await settled(id); + expect(n).toBe(1); + expect(items.map((i) => i.viewsAuto?.state)).toEqual(['failed', 'failed']); + expect(duelViewsNote(items[1]!)).toContain('00:00까지 쉬어요'); +}); diff --git a/apps/server/src/quizzes/duel-views.ts b/apps/server/src/quizzes/duel-views.ts index 815eae4..988591e 100644 --- a/apps/server/src/quizzes/duel-views.ts +++ b/apps/server/src/quizzes/duel-views.ts @@ -6,7 +6,7 @@ */ import type { Database } from 'bun:sqlite'; import { PENDING_VIEWS, type DuelViews } from '@mq/shared'; -import { CookieError, YtMusic, type SongPair } from '../collector/ytmusic'; +import { CookieError, RateLimited, YtMusic, type SongPair } from '../collector/ytmusic'; import type { LogFn } from '../server'; export interface ViewsSource { @@ -123,7 +123,7 @@ export class DuelViewsJob { try { src ??= await this.source(); result = await fetchDuelViews(row.video_id, src).catch(async (e) => { - if (e instanceof CookieError) throw e; + if (e instanceof CookieError || e instanceof RateLimited) throw e; await sleep(this.opts.retryMs ?? 3000); return fetchDuelViews(row!.video_id, src!); }); @@ -135,13 +135,20 @@ export class DuelViewsJob { this.opts.log?.('warn', 'duel views: cookie', { message: e.message }); break; } + if (e instanceof RateLimited) { + // the same for every waiting song: they wait for 다시 불러오기 once the pause is over + const failed = JSON.stringify(failure(`${e.message} — 그 뒤에 다시 불러와 주세요`)); + this.db.query(`UPDATE custom_quiz_items SET views_auto = ? WHERE ${PENDING}`).run(failed); + this.opts.log?.('warn', 'duel views: rate limited', { message: e.message }); + break; + } result = failure(`조회수를 불러오지 못했어요: ${(e as Error).message}`.slice(0, 300)); } const views = result.state === 'done' ? totalViews(result) : null; if (views !== null) write.run(views, JSON.stringify(result), row.id, row.video_id); else this.db.query(`UPDATE custom_quiz_items SET views_auto = ? WHERE id = ? AND video_id = ? AND ${PENDING}`).run(JSON.stringify(result), row.id, row.video_id); changed = true; - await sleep(this.opts.delayMs ?? 500); + await sleep(this.opts.delayMs ?? 2000); } if (changed) this.opts.onChange?.(); } diff --git a/docs/08-youtube-playback.md b/docs/08-youtube-playback.md index 079fc57..c914cd9 100644 --- a/docs/08-youtube-playback.md +++ b/docs/08-youtube-playback.md @@ -38,13 +38,15 @@ ### 쿠키 갱신 (`collector/cookie-keeper.ts`, 2026-10-29) - 원인: 구글 로그인 쿠키의 `__Secure-1PSIDTS`·`__Secure-3PSIDTS`·`*SIDCC`는 브라우저 탭이 10분마다 `accounts.youtube.com/RotateCookies`를 불러 새 값으로 바꾼다. 서버에 복사한 쿠키는 이 값을 못 받아 며칠 뒤 로그아웃됐다(10/19 쿠키 → 10/28 08:10 "로그인이 풀렸어요"). 쿠키를 뽑은 브라우저가 같은 세션을 계속 쓰면 더 빨리 끊긴다 - 서버(index.ts, `YT_COOKIE_PATH`가 있을 때)가 10분마다 같은 요청(`POST accounts.youtube.com/RotateCookies`, origin `https://www.youtube.com`, 본문 `[1,"-숫자"]`)을 보내고 응답 `Set-Cookie`의 youtube.com 값을 쿠키 파일에 그 형식 그대로(헤더/cookies.txt) 덮어쓴다. 그다음 유튜브 뮤직 노래 검색으로 로그인 상태인지 확인 -- 갱신 401/403 또는 검색이 로그아웃 상태 → "로그인이 풀렸어요"(관리자 → 수집 맨 위). 새 쿠키를 넣을 때까지 갱신을 멈추고, 파일이 바뀌면 1분 안에 다시 확인. 네트워크 오류·429는 10분 뒤 재시도 +- 갱신 401/403 또는 검색이 로그아웃 상태 → "로그인이 풀렸어요"(관리자 → 수집 맨 위). 새 쿠키를 넣을 때까지 갱신을 멈추고, 파일이 바뀌면 1분 안에 다시 확인. 네트워크 오류는 10분 뒤 재시도 +- 429(요청이 너무 많음, 2026-11-03): 갱신·로그인 확인·노래 검색·조회수 짝 찾기·연령제한 음원 받기 중 어디서든 429(yt-dlp는 차단 신호)가 나오면 `<쿠키 파일>.cooldown`에 지금+3시간을 적는다. 그동안 쿠키를 쓰는 요청은 보내지 않는다(`RateLimited`). 원곡 찾기·연령제한 작업은 멈추고, 조회수는 "HH:MM까지 쉬어요" 실패로 남아 다시 불러오기를 기다린다. 갱신도 쉬는데, 새 쿠키 파일이 들어오면 바로 확인한다. 11/03에는 원곡 찾기 567곡(2초 간격) 직후 갱신이 429를 받았고, 10분 뒤 401로 세션이 끝났다 +- 일부만 복사한 쿠키: 다른 사이트(퍼간 플레이어)가 보낸 요청의 쿠키에는 `__Secure-3P*`만 있고 `SAPISID`·`__Secure-1PSID`·`SID`가 없다. `SAPISID`는 `__Secure-3PAPISID`와 같은 값이라 읽을 때 채운다(파일은 그대로). 로그인 값은 `SAPISID`(또는 채울 값)와 `__Secure-1PSID`/`__Secure-3PSID` 중 하나가 있으면 된다 - 갱신은 서버 한 곳만(세션 하나에 갱신하는 쪽이 둘이면 서로의 값을 무효로 만든다). 수집기는 파일을 요청마다 새로 읽고(`YtMusic`은 파일이 바뀌면 다시 읽음, age-audio는 곡마다), yt-dlp가 돌려준 쿠키 중 값이 바뀐 것은 파일에 합친다(받는 동안 파일이 바뀌었으면 — 서버가 갱신했거나 새 쿠키가 들어왔으면 — 파일 쪽이 더 새 값이라 yt-dlp 값은 버린다) - 잰 것(10/29, 이미 로그아웃된 쿠키): `accounts.youtube.com/RotateCookies` 200 + `SIDCC`만 새로 줌(PSIDTS 없음), 검색은 로그아웃 → 상태는 검색으로 판단해야 한다. 쿠키 없이 403, `accounts.google.com/RotateCookies`는 youtube 쿠키로 401 - 쿠키 받는 법: 크롬 시크릿 창에서 로그인 → music.youtube.com → 쿠키 내보내기 → 로그아웃하지 말고 창을 닫는다(브라우저가 그 세션을 다시 쓰지 않게). 비밀번호 변경·"모든 기기 로그아웃"·구글 보안 확인이 있으면 다시 끊긴다 ### 요청 속도 -- 곡당 검색 1~3회, 2초 간격, 하루 최대 3,000곡. 차단 신호(HTTP 429, 봇 확인 페이지) 보이면 자동 중단 후 다음 날 재개 +- 곡당 검색 1~3회, 5초 간격(2026-11-03, 2초에서 늘림), 하루 최대 300곡(cron). 429가 한 번이라도 나오면 바로 멈추고 3시간 동안 쿠키를 쓰는 요청을 하지 않음(위 429), 다음 날 cron에서 재개 ## 2. 재생 방식 **확정: B안(숨긴 IFrame 플레이어) + 서버 기준 동기화**. 자세한 동기화 규칙은 12-decisions.md Q1. 아래 A안은 검토 기록으로만 남긴다. diff --git a/docs/09-deploy-ops.md b/docs/09-deploy-ops.md index eb82447..4b49990 100644 --- a/docs/09-deploy-ops.md +++ b/docs/09-deploy-ops.md @@ -47,6 +47,6 @@ - .9 `/home/claude/projects/music_quiz` (gitea main 클론), 컨테이너 `music_quiz-app-1`, 볼륨 `music_quiz_app-data`, `deploy/.env` 권한 600 - NPMplus 프록시 호스트 id 14 → `192.168.10.9:2974`, 고급 설정은 `deploy/npmplus-advanced.conf`, 인증서는 `deploy/cert.sh`(다시 돌려도 안전) - 백업(매일 `VACUUM INTO`)은 노래 DB가 생기는 3단계에서 붙인다 -- 수집 cron(.9 crontab, 컨테이너 안에서 `docker exec`): 02:00 차트, 02:20 멜론 곡 정보 1,000곡, 03:15 하이라이트 400곡(4초 간격), 03:55 난이도, 08:10 유튜브 원곡 찾기 3,000곡(04:00~08:00은 봇이 Claude를 부르지 않는 시간이라 피함), 12:30 소리 크기 재기 3,000곡(3초 간격, 새로 확정된 곡), 15:40 연령제한 곡 음원 받기 100곡(`age-audio`, 3초 간격). 로그 `~/music_quiz-collect.log`, `~/music_quiz-match.log`. 관리자 "수집" 화면에서 따로 실행·멈추기 가능 +- 수집 cron(.9 crontab, 컨테이너 안에서 `docker exec`): 02:00 차트, 02:20 멜론 곡 정보 1,000곡, 03:15 하이라이트 400곡(4초 간격), 03:55 난이도, 08:10 유튜브 원곡 찾기 300곡(5초 간격, 429면 멈춤 — 2026-11-03에 3,000곡에서 줄임, 04:00~08:00은 봇이 Claude를 부르지 않는 시간이라 피함), 12:30 소리 크기 재기 3,000곡(3초 간격, 새로 확정된 곡), 15:40 연령제한 곡 음원 받기 100곡(`age-audio`, 3초 간격). 로그 `~/music_quiz-collect.log`, `~/music_quiz-match.log`. 관리자 "수집" 화면에서 따로 실행·멈추기 가능 - 9단계 전 DB 백업: `~/backups/music_quiz/app-before-stage9-*.db` diff --git a/docs/12-decisions.md b/docs/12-decisions.md index ec661e8..c09f42f 100644 --- a/docs/12-decisions.md +++ b/docs/12-decisions.md @@ -114,6 +114,7 @@ - 사용자가 준 `ytm_cookie.txt` 를 호스트 비밀 위치 `/home/claude/.config/music_quiz/ytm_cookie.txt`(권한 600)에 보관. 리포에는 넣지 않는다 - 쓰는 곳: 노래 수집기의 YT Music 노래 검색(music_bot_v2 방식 `SAPISIDHASH`), 관리자 "곡 추가" 자동 채우기, 연령제한 음원 받기. 컨테이너에는 폴더째 쓰기 가능으로 연결(`YT_COOKIE_PATH`) - (2026-10-29) 서버가 브라우저처럼 10분마다 쿠키 세션 값을 갱신해 파일에 쓴다(08 §1 쿠키 갱신). 쿠키는 시크릿 창 전용 세션으로 받는다. OAuth는 yt-dlp가 지원하지 않아 쓰지 않음 +- (2026-11-03, 사용자 확인 "3개다 적용해줘") 10/30 쿠키가 11/03 08:10 원곡 찾기 567곡(2초 간격) 직후 갱신 429 → 10분 뒤 401로 끊김. 평소 크롬과 같은 세션을 함께 쓴 것도 원인으로 봄. ① 쿠키는 시크릿 창(가능하면 서버 전용 계정)으로 새로 받는다 ② 원곡 찾기 5초 간격, cron 하루 300곡, 429가 한 번이라도 나오면 그 작업은 멈추고 쿠키를 쓰는 모든 작업(갱신·검색·조회수·연령제한 음원)이 3시간 쉰다(`<쿠키 파일>.cooldown`) ③ 다른 사이트 요청에서 복사해 `__Secure-3P*`만 있는 쿠키도 받는다(SAPISID는 같은 값인 `__Secure-3PAPISID`로 채움) - music_bot_v2 쿠키와는 분리해서 쓴다 → 차단돼도 봇에는 영향 없음 - 쿠키 만료(로그인 필요 응답) 감지 시 관리자 → 수집 맨 위에 표시(디스코드 알림은 하지 않음, 사용자 결정 2026-10-29)