From e2a9e81c2cdee756f54cf895f99124556624ce2a Mon Sep 17 00:00:00 2001 From: EJClaw Date: Thu, 29 Oct 2026 14:44:25 +0900 Subject: [PATCH] =?UTF-8?q?=EC=9C=A0=ED=8A=9C=EB=B8=8C=20=EB=AE=A4?= =?UTF-8?q?=EC=A7=81=20=EC=BF=A0=ED=82=A4=20=EC=9E=90=EB=8F=99=20=EA=B0=B1?= =?UTF-8?q?=EC=8B=A0:=20=EC=84=9C=EB=B2=84=EA=B0=80=20=EB=B8=8C=EB=9D=BC?= =?UTF-8?q?=EC=9A=B0=EC=A0=80=20=ED=83=AD=EC=B2=98=EB=9F=BC=2010=EB=B6=84?= =?UTF-8?q?=EB=A7=88=EB=8B=A4=20accounts.youtube.com/RotateCookies?= =?UTF-8?q?=EB=A5=BC=20=EB=B6=88=EB=9F=AC=20=EC=83=88=20=5F=5FSecure-*PSID?= =?UTF-8?q?TS=C2=B7*SIDCC=20=EA=B0=92=EC=9D=84=20=EC=BF=A0=ED=82=A4=20?= =?UTF-8?q?=ED=8C=8C=EC=9D=BC=EC=97=90=20=EA=B7=B8=20=ED=98=95=EC=8B=9D=20?= =?UTF-8?q?=EA=B7=B8=EB=8C=80=EB=A1=9C=20=EC=94=80(collector/cookie-keeper?= =?UTF-8?q?.ts,=20cookie-jar.ts),=20=EC=9D=B4=EC=96=B4=EC=84=9C=20?= =?UTF-8?q?=EB=85=B8=EB=9E=98=20=EA=B2=80=EC=83=89=EC=9C=BC=EB=A1=9C=20?= =?UTF-8?q?=EB=A1=9C=EA=B7=B8=EC=9D=B8=20=EC=83=81=ED=83=9C=20=ED=99=95?= =?UTF-8?q?=EC=9D=B8.=20=EB=A7=8C=EB=A3=8C(=EA=B0=B1=EC=8B=A0=20401/403?= =?UTF-8?q?=C2=B7=EB=A1=9C=EA=B7=B8=EC=95=84=EC=9B=83=20=EA=B2=80=EC=83=89?= =?UTF-8?q?)=EB=8A=94=20=EA=B4=80=EB=A6=AC=EC=9E=90=20=E2=86=92=20?= =?UTF-8?q?=EC=88=98=EC=A7=91=20=EB=A7=A8=20=EC=9C=84=EC=97=90=20=ED=91=9C?= =?UTF-8?q?=EC=8B=9C=ED=95=98=EA=B3=A0=20=EC=83=88=20=ED=8C=8C=EC=9D=BC?= =?UTF-8?q?=EC=9D=B4=20=EB=93=A4=EC=96=B4=EC=98=AC=20=EB=95=8C=EA=B9=8C?= =?UTF-8?q?=EC=A7=80=20=EA=B0=B1=EC=8B=A0=20=EB=A9=88=EC=B6=A4(=ED=8C=8C?= =?UTF-8?q?=EC=9D=BC=EC=9D=B4=20=EB=B0=94=EB=80=8C=EB=A9=B4=201=EB=B6=84?= =?UTF-8?q?=20=EC=95=88=EC=97=90=20=EB=8B=A4=EC=8B=9C=20=ED=99=95=EC=9D=B8?= =?UTF-8?q?).=20=EC=9B=90=EA=B3=A1=20=EC=B0=BE=EA=B8=B0=20YtMusic=EC=9D=80?= =?UTF-8?q?=20=ED=8C=8C=EC=9D=BC=EC=9D=B4=20=EB=B0=94=EB=80=8C=EB=A9=B4=20?= =?UTF-8?q?=EB=8B=A4=EC=8B=9C=20=EC=9D=BD=EA=B3=A0,=20=EC=97=B0=EB=A0=B9?= =?UTF-8?q?=EC=A0=9C=ED=95=9C=20=EC=9D=8C=EC=9B=90=20=EB=B0=9B=EA=B8=B0?= =?UTF-8?q?=EB=8A=94=20=EA=B3=A1=EB=A7=88=EB=8B=A4=20=EC=83=88=EB=A1=9C=20?= =?UTF-8?q?=EC=9D=BD=EA=B3=A0=20yt-dlp=EA=B0=80=20=EB=B0=94=EA=BE=BC=20?= =?UTF-8?q?=EA=B0=92=EC=9D=84=20=ED=8C=8C=EC=9D=BC=EC=97=90=20=ED=95=A9?= =?UTF-8?q?=EC=B9=A8.=20compose:=20=EC=BF=A0=ED=82=A4=20=ED=8F=B4=EB=8D=94?= =?UTF-8?q?=EB=A5=BC=20=EC=93=B0=EA=B8=B0=20=EA=B0=80=EB=8A=A5=EC=9C=BC?= =?UTF-8?q?=EB=A1=9C=20=EC=97=B0=EA=B2=B0(YT=5FCOOKIE=5FPATH=3D/run/secret?= =?UTF-8?q?s/music=5Fquiz/ytm=5Fcookie.txt).=20=EB=8B=A8=EC=9C=84=C2=B7?= =?UTF-8?q?=EA=B4=80=EB=A6=AC=EC=9E=90=20API=C2=B7e2e=20=ED=85=8C=EC=8A=A4?= =?UTF-8?q?=ED=8A=B8,=20docs=2008=C2=B709=C2=B712?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- apps/server/src/admin-catalog.test.ts | 9 + apps/server/src/collector/age-audio.ts | 40 ++--- apps/server/src/collector/cookie-jar.ts | 108 ++++++++++++ .../src/collector/cookie-keeper.test.ts | 162 ++++++++++++++++++ apps/server/src/collector/cookie-keeper.ts | 129 ++++++++++++++ apps/server/src/collector/ytmusic.ts | 22 ++- apps/server/src/http/admin-catalog.ts | 5 +- apps/server/src/http/admin-quizzes.ts | 2 +- apps/server/src/http/admin.ts | 3 + apps/server/src/index.ts | 7 +- apps/server/src/server.ts | 3 + apps/web/src/pages/admin/AdminMore.tsx | 33 +++- deploy/.env.example | 2 +- deploy/docker-compose.proxy.yml | 7 +- docs/08-youtube-playback.md | 10 +- docs/09-deploy-ops.md | 2 +- docs/12-decisions.md | 5 +- e2e/admin.e2e.ts | 10 ++ 18 files changed, 521 insertions(+), 38 deletions(-) create mode 100644 apps/server/src/collector/cookie-jar.ts create mode 100644 apps/server/src/collector/cookie-keeper.test.ts create mode 100644 apps/server/src/collector/cookie-keeper.ts diff --git a/apps/server/src/admin-catalog.test.ts b/apps/server/src/admin-catalog.test.ts index d1dc002..b2ea4ff 100644 --- a/apps/server/src/admin-catalog.test.ts +++ b/apps/server/src/admin-catalog.test.ts @@ -193,6 +193,15 @@ describe('admin artists, genres, stats, rooms, collect, room defaults', () => { expect(runs).toEqual([{ kind: 'match-youtube', limit: 50 }]); }); + test('collect: the cookie keeper status replaces the guess from the last job log', async () => { + let state: 'ok' | 'expired' = 'ok'; + const { call } = await admin({ cookieStatus: () => ({ state, message: state === 'ok' ? null : '갱신 401', checkedAt: 1, rotatedAt: 1 }) }); + s.db.query("INSERT INTO collect_jobs (kind, status, total, started_at, log) VALUES ('match-youtube', 'stopped', 0, 1, 'stopped: 유튜브 뮤직 로그인이 풀렸어요')").run(); + expect((await call('/collect')).body).toMatchObject({ cookieProblem: false, cookie: { state: 'ok' } }); + state = 'expired'; + expect((await call('/collect')).body).toMatchObject({ cookieProblem: true, cookie: { state: 'expired', message: '갱신 401' } }); + }); + test('collect control: one run per kind, stop marks a dead process stopped', async () => { s = boot(); const { collectorControl } = await import('./collector/control'); diff --git a/apps/server/src/collector/age-audio.ts b/apps/server/src/collector/age-audio.ts index c0ea840..d95fc12 100644 --- a/apps/server/src/collector/age-audio.ts +++ b/apps/server/src/collector/age-audio.ts @@ -9,34 +9,17 @@ * Age-restricted songs are found by the loudness / heatmap jobs (their anonymous page) and by step 1. */ import type { Database } from 'bun:sqlite'; -import { existsSync, mkdirSync, mkdtempSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { audioFile } from '../songs/audio'; import { Catalog, Job, sleep } from './catalog'; import { Blocked } from './charts'; +import { cookieForms, jarChanges, mergeCookies, writeCookieFile } from './cookie-jar'; import { parseHeatmap, pickHighlight } from './heatmap'; import { fetchWatchPage, isAgeGated, parseLoudness, saveLoudness } from './loudness'; -/** The cookie file holds either a request header (`a=1; b=2`) or a Netscape cookies.txt; returns both forms. */ -export function cookieForms(raw: string): { header: string; netscape: string } { - const text = raw.trim(); - if (text.includes('\t')) { - const rows = text - .split(/\r?\n/) - .filter((l) => l && (!l.startsWith('#') || l.startsWith('#HttpOnly_'))) - .map((l) => l.split('\t')) - .filter((p) => p.length >= 7 && /youtube\.com$/.test(p[0]!)); - return { header: rows.map((p) => `${p[5]}=${p[6]!.trim()}`).join('; '), netscape: `${text}\n` }; - } - const exp = Math.floor(Date.now() / 1000) + 365 * 86400; - const pairs = text - .split(/;\s*/) - .map((p) => [p.slice(0, p.indexOf('=')).trim(), p.slice(p.indexOf('=') + 1).trim()] as const) - .filter(([k]) => k); - const lines = pairs.map(([k, v]) => ['.youtube.com', 'TRUE', '/', 'TRUE', exp, k, v].join('\t')); - return { header: pairs.map(([k, v]) => `${k}=${v}`).join('; '), netscape: ['# Netscape HTTP Cookie File', ...lines, ''].join('\n') }; -} +export { cookieForms } from './cookie-jar'; /** yt-dlp: the best m4a audio-only stream of the video into `out` (no ffmpeg in the image: browsers play the DASH m4a as is). */ export async function ytDlpAudio(videoId: string, out: string, cookieFile: string, bin = process.env.YTDLP_BIN ?? 'yt-dlp'): Promise { @@ -132,12 +115,21 @@ export async function collectAgeAudio(db: Database, dir: string, opts: AgeAudioO if (todo.length && (!fetchSignedIn || !download)) { const path = opts.cookiePath ?? process.env.YT_COOKIE_PATH ?? '/home/claude/.config/music_quiz/ytm_cookie.txt'; if (!existsSync(path)) throw new CookieProblem(`쿠키 파일이 없어요: ${path}`); - const c = cookieForms(await Bun.file(path).text()); tmp = mkdtempSync(join(tmpdir(), 'mq-cookie-')); const cookieFile = join(tmp, 'cookies.txt'); - writeFileSync(cookieFile, c.netscape, { mode: 0o600 }); - fetchSignedIn ??= (id) => fetchWatchPage(id, c.header); - download ??= (id, out) => ytDlpAudio(id, out, cookieFile); + // the file is read for every song: the server keeps rotating its values (cookie-keeper.ts) + fetchSignedIn ??= async (id) => fetchWatchPage(id, cookieForms(readFileSync(path, 'utf8')).header); + download ??= async (id, out) => { + const given = cookieForms(readFileSync(path, 'utf8')).netscape; + writeFileSync(cookieFile, given, { mode: 0o600 }); + try { + await ytDlpAudio(id, out, cookieFile); + } finally { + // yt-dlp writes the jar back: values YouTube changed on the way go into the file, so the session stays one + const changes = existsSync(cookieFile) ? jarChanges(given, readFileSync(cookieFile, 'utf8')) : []; + if (changes.length) writeCookieFile(path, mergeCookies(readFileSync(path, 'utf8'), changes)); + } + }; } for (const s of todo) { try { diff --git a/apps/server/src/collector/cookie-jar.ts b/apps/server/src/collector/cookie-jar.ts new file mode 100644 index 0000000..ea1c497 --- /dev/null +++ b/apps/server/src/collector/cookie-jar.ts @@ -0,0 +1,108 @@ +/** + * The YouTube cookie file (YT_COOKIE_PATH): a request header (`a=1; b=2`) or a Netscape cookies.txt. + * Google keeps changing some of its values (__Secure-*PSIDTS, *SIDCC); the server writes them back here (cookie-keeper.ts), + * so every reader takes the file fresh instead of keeping a copy. + */ +import { writeFileSync } from 'node:fs'; + +/** Returns both forms of the file: the header for fetch, the cookies.txt for yt-dlp. */ +export function cookieForms(raw: string): { header: string; netscape: string } { + const text = raw.trim(); + if (text.includes('\t')) { + const rows = netscapeRows(text).filter((p) => /youtube\.com$/.test(p[0]!)); + return { header: rows.map((p) => `${p[5]}=${p[6]!.trim()}`).join('; '), netscape: `${text}\n` }; + } + const exp = Math.floor(Date.now() / 1000) + 365 * 86400; + const pairs = headerPairs(text); + const lines = pairs.map(([k, v]) => ['.youtube.com', 'TRUE', '/', 'TRUE', exp, k, v].join('\t')); + return { header: pairs.map(([k, v]) => `${k}=${v}`).join('; '), netscape: ['# Netscape HTTP Cookie File', ...lines, ''].join('\n') }; +} + +function headerPairs(text: string): [string, string][] { + return text + .split(/;\s*/) + .map((p) => [p.slice(0, p.indexOf('=')).trim(), p.slice(p.indexOf('=') + 1).trim()] as [string, string]) + .filter(([k]) => k); +} + +/** Data rows of a cookies.txt (`#HttpOnly_` rows count; the prefix is dropped from the domain). */ +function netscapeRows(text: string): string[][] { + return text + .split(/\r?\n/) + .filter((l) => l && (!l.startsWith('#') || l.startsWith('#HttpOnly_'))) + .map((l) => l.replace(/^#HttpOnly_/, '').split('\t')) + .filter((p) => p.length >= 7); +} + +export interface CookieUpdate { + name: string; + value: string; + /** Unix seconds, when the server gave one. */ + expires: number | null; +} + +/** `Set-Cookie` lines of a youtube.com response → the cookies to store (deletions and other domains are left out). */ +export function parseSetCookies(lines: string[], now = Date.now()): CookieUpdate[] { + const out: CookieUpdate[] = []; + for (const line of lines) { + const [pair, ...attrs] = line.split(';'); + const i = pair!.indexOf('='); + if (i <= 0) continue; + const name = pair!.slice(0, i).trim(); + const value = pair!.slice(i + 1).trim(); + let expires: number | null = null; + let domain: string | null = null; + for (const a of attrs) { + const [k, ...v] = a.trim().split('='); + const val = v.join('=').trim(); + if (/^expires$/i.test(k!)) expires = Math.floor(Date.parse(val) / 1000) || null; + else if (/^max-age$/i.test(k!)) expires = Math.floor(now / 1000) + Number(val); + else if (/^domain$/i.test(k!)) domain = val.toLowerCase(); + } + if (domain && !/(^|\.)youtube\.com$/.test(domain)) continue; + if (!value || (expires !== null && expires * 1000 <= now)) continue; + out.push({ name, value, expires }); + } + return out; +} + +/** The file with the new values, in the file's own form. Unknown cookies are added. */ +export function mergeCookies(raw: string, updates: CookieUpdate[]): string { + if (!updates.length) return raw; + const text = raw.trim(); + if (!text.includes('\t')) { + const pairs = headerPairs(text); + for (const u of updates) { + const p = pairs.find(([k]) => k === u.name); + if (p) p[1] = u.value; + else pairs.push([u.name, u.value]); + } + return `${pairs.map(([k, v]) => `${k}=${v}`).join('; ')}\n`; + } + const left = new Map(updates.map((u) => [u.name, u])); + const lines = text.split(/\r?\n/).map((l) => { + if (!l || (l.startsWith('#') && !l.startsWith('#HttpOnly_'))) return l; + const p = l.split('\t'); + const u = p.length >= 7 && /youtube\.com$/.test(p[0]!.replace(/^#HttpOnly_/, '')) ? left.get(p[5]!) : undefined; + if (!u) return l; + left.delete(u.name); + p[6] = u.value; + if (u.expires) p[4] = String(u.expires); + return p.join('\t'); + }); + for (const u of left.values()) lines.push(['.youtube.com', 'TRUE', '/', 'TRUE', u.expires ?? Math.floor(Date.now() / 1000) + 365 * 86400, u.name, u.value].join('\t')); + return `${lines.join('\n')}\n`; +} + +/** Cookies yt-dlp changed in its cookies.txt (it writes the jar back when it exits). */ +export function jarChanges(before: string, after: string): CookieUpdate[] { + const old = new Map(netscapeRows(before).map((p) => [`${p[0]}\t${p[5]}`, p[6]])); + return netscapeRows(after) + .filter((p) => /youtube\.com$/.test(p[0]!) && p[6] && old.has(`${p[0]}\t${p[5]}`) && old.get(`${p[0]}\t${p[5]}`) !== p[6]) + .map((p) => ({ name: p[5]!, value: p[6]!.trim(), expires: Number(p[4]) || null })); +} + +/** Rewrites the file in place: it is a single-file bind mount in the container, so a rename would not reach the host. */ +export function writeCookieFile(path: string, text: string): void { + writeFileSync(path, text); +} diff --git a/apps/server/src/collector/cookie-keeper.test.ts b/apps/server/src/collector/cookie-keeper.test.ts new file mode 100644 index 0000000..96ca13d --- /dev/null +++ b/apps/server/src/collector/cookie-keeper.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, test } from 'bun:test'; +import { mkdtempSync, readFileSync, statSync, utimesSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { cookieForms, jarChanges, mergeCookies, parseSetCookies } from './cookie-jar'; +import { CookieKeeper } from './cookie-keeper'; +import { CookieError, YtMusic } from './ytmusic'; + +const NOW = Date.parse('2026-10-29T05:00:00Z'); +const HEADER = 'SID=s1; SAPISID=sap; __Secure-1PSID=p1; __Secure-1PSIDTS=old1; __Secure-3PSIDTS=old3; SIDCC=c0\n'; +const ROTATED = [ + '__Secure-1PSIDTS=new1; expires=Fri, 29-Oct-2027 05:00:00 GMT; path=/; domain=.youtube.com; Secure; HttpOnly', + '__Secure-3PSIDTS=new3; expires=Fri, 29-Oct-2027 05:00:00 GMT; path=/; domain=.youtube.com; Secure; HttpOnly; SameSite=none', + 'SIDCC=c1; expires=Fri, 29-Oct-2027 05:00:00 GMT; path=/; domain=.youtube.com; priority=high', +]; + +function tempFile(text: string): string { + const p = join(mkdtempSync(join(tmpdir(), 'mq-keeper-')), 'ytm_cookie.txt'); + writeFileSync(p, text); + return p; +} +const rotateRes = (status: number, setCookies: string[] = []) => { + const h = new Headers(); + for (const c of setCookies) h.append('set-cookie', c); + return new Response(`)]}'\n\n[["identity.hfcr",600]]`, { status, headers: h }); +}; + +describe('쿠키 파일 갱신', () => { + test('Set-Cookie → youtube.com values only; deletions are left out', () => { + const u = parseSetCookies([...ROTATED, 'NID=z; domain=.google.com', 'LOGIN_INFO=; expires=Thu, 01-Jan-1970 00:00:01 GMT; domain=.youtube.com', 'YSC=y; path=/'], NOW); + expect(u.map((x) => `${x.name}=${x.value}`)).toEqual(['__Secure-1PSIDTS=new1', '__Secure-3PSIDTS=new3', 'SIDCC=c1', 'YSC=y']); + expect(u[0]!.expires).toBe(Date.parse('2027-10-29T05:00:00Z') / 1000); + }); + + test('merges into a header file, keeping the order and adding new cookies', () => { + const out = mergeCookies(HEADER, parseSetCookies([...ROTATED, 'YSC=y'], NOW)); + expect(out).toBe('SID=s1; SAPISID=sap; __Secure-1PSID=p1; __Secure-1PSIDTS=new1; __Secure-3PSIDTS=new3; SIDCC=c1; YSC=y\n'); + }); + + test('merges into a cookies.txt: value and expiry of youtube.com rows (#HttpOnly_ too), other domains untouched', () => { + const txt = [ + '# Netscape HTTP Cookie File', + '#HttpOnly_.youtube.com\tTRUE\t/\tTRUE\t1800000000\t__Secure-1PSIDTS\told1', + '.youtube.com\tTRUE\t/\tFALSE\t1800000000\tSIDCC\tc0', + '.google.com\tTRUE\t/\tFALSE\t1800000000\tSIDCC\tg0', + '', + ].join('\n'); + const out = mergeCookies(txt, parseSetCookies(ROTATED, NOW)); + expect(out).toContain('#HttpOnly_.youtube.com\tTRUE\t/\tTRUE\t1824786000\t__Secure-1PSIDTS\tnew1'); + expect(out).toContain('.youtube.com\tTRUE\t/\tFALSE\t1824786000\tSIDCC\tc1'); + expect(out).toContain('.google.com\tTRUE\t/\tFALSE\t1800000000\tSIDCC\tg0'); + expect(out).toContain('\t__Secure-3PSIDTS\tnew3'); + expect(cookieForms(out).header).toBe('__Secure-1PSIDTS=new1; SIDCC=c1; __Secure-3PSIDTS=new3'); + }); + + test('yt-dlp jar written back → only the values it changed', () => { + const given = cookieForms(HEADER).netscape; + const after = given.replace('\tSIDCC\tc0', '\tSIDCC\tc9').replace('# Netscape HTTP Cookie File', '# Netscape HTTP Cookie File\n.youtube.com\tTRUE\t/\tTRUE\t0\tYSC\tnew'); + expect(jarChanges(given, after).map((u) => `${u.name}=${u.value}`)).toEqual(['SIDCC=c9']); + expect(jarChanges(given, given)).toEqual([]); + }); +}); + +describe('CookieKeeper', () => { + test('rotates like a browser tab: new values go into the file, then the login is checked with them', async () => { + const path = tempFile(HEADER); + const calls: { url: string; init: RequestInit }[] = []; + const checked: string[] = []; + const k = new CookieKeeper(path, { + fetch: (async (url: string, init: RequestInit) => (calls.push({ url, init }), rotateRes(200, ROTATED))) as unknown as typeof fetch, + checkLogin: async (h) => void checked.push(h), + }); + await k.run(); + expect(calls[0]!.url).toBe('https://accounts.youtube.com/RotateCookies'); + expect((calls[0]!.init.headers as Record).cookie).toBe(HEADER.trim()); + expect((calls[0]!.init.headers as Record).origin).toBe('https://www.youtube.com'); + expect(readFileSync(path, 'utf8')).toBe('SID=s1; SAPISID=sap; __Secure-1PSID=p1; __Secure-1PSIDTS=new1; __Secure-3PSIDTS=new3; SIDCC=c1\n'); + expect(checked[0]).toContain('__Secure-1PSIDTS=new1'); + expect(k.status.state).toBe('ok'); + expect(k.status.rotatedAt).not.toBeNull(); + }); + + test('an ended session waits for a new cookie file, then is checked again right away', async () => { + const path = tempFile(HEADER); + let status = 401; + let n = 0; + const k = new CookieKeeper(path, { + fetch: (async () => (n++, rotateRes(status, status === 200 ? ROTATED : []))) as unknown as typeof fetch, + checkLogin: async () => {}, + everyMs: 1000, + }); + await k.tick(); + expect(k.status.state).toBe('expired'); + await k.tick(Date.now() + 60_000); + expect(n).toBe(1); // no more calls with the dead cookie + status = 200; + writeFileSync(path, HEADER.replace('p1', 'p2')); + const t = new Date(Date.now() + 5000); + utimesSync(path, t, t); + await k.tick(); + expect(n).toBe(2); + expect(k.status.state).toBe('ok'); + expect(readFileSync(path, 'utf8')).toContain('__Secure-1PSID=p2; __Secure-1PSIDTS=new1'); + }); + + test('a logged-out search is an ended session even when the rotation answered 200; network trouble is retried later', async () => { + const path = tempFile(HEADER); + const k = new CookieKeeper(path, { + fetch: (async () => rotateRes(200, [ROTATED[2]!])) as unknown as typeof fetch, + checkLogin: async () => { + throw new CookieError('유튜브 뮤직 로그인이 풀렸어요 — 쿠키 갱신 필요'); + }, + }); + await k.run(); + expect(k.status).toMatchObject({ state: 'expired', message: '유튜브 뮤직 로그인이 풀렸어요 — 쿠키 갱신 필요', rotatedAt: null }); + + let n = 0; + const k2 = new CookieKeeper(tempFile(HEADER), { + fetch: (async () => { + n++; + throw new Error('ECONNRESET'); + }) as unknown as typeof fetch, + checkLogin: async () => {}, + everyMs: 10 * 60_000, + }); + await k2.tick(); + expect(k2.status.state).toBe('error'); + await k2.tick(Date.now() + 60_000); + expect(n).toBe(1); + await k2.tick(Date.now() + 11 * 60_000); + expect(n).toBe(2); + }); + + test('no file / no login cookie', async () => { + const dir = mkdtempSync(join(tmpdir(), 'mq-keeper-')); + const k = new CookieKeeper(join(dir, 'none.txt'), { fetch: (async () => rotateRes(200)) as unknown as typeof fetch }); + await k.run(); + expect(k.status.state).toBe('missing'); + const k2 = new CookieKeeper(tempFile('YSC=a; PREF=b'), { fetch: (async () => rotateRes(200)) as unknown as typeof fetch }); + await k2.run(); + expect(k2.status.state).toBe('expired'); + }); +}); + +describe('YtMusic reads the cookie file again when it changes', () => { + test('a long 원곡 찾기 run uses the rotated values', async () => { + const path = tempFile(HEADER); + const sent: string[] = []; + const fake = (async (_u: string, init: RequestInit) => { + sent.push((init.headers as Record).cookie ?? ''); + return Response.json({ responseContext: { serviceTrackingParams: [{ params: [{ key: 'logged_in', value: '1' }] }] } }); + }) as unknown as typeof fetch; + const ytm = new YtMusic('', fake, path); + await ytm.searchSongs('a'); + writeFileSync(path, mergeCookies(HEADER, parseSetCookies(ROTATED, NOW))); + const t = new Date(statSync(path).mtimeMs + 5000); + utimesSync(path, t, t); + await ytm.searchSongs('b'); + expect(sent[0]).toContain('__Secure-1PSIDTS=old1'); + expect(sent[1]).toContain('__Secure-1PSIDTS=new1'); + }); +}); diff --git a/apps/server/src/collector/cookie-keeper.ts b/apps/server/src/collector/cookie-keeper.ts new file mode 100644 index 0000000..e0f0d8c --- /dev/null +++ b/apps/server/src/collector/cookie-keeper.ts @@ -0,0 +1,129 @@ +/** + * Keeps the YouTube Music cookie signed in (docs/08 §1 쿠키 갱신). A browser tab calls accounts.youtube.com/RotateCookies + * every 10 minutes and Google answers with new __Secure-*PSIDTS / *SIDCC values; a copied cookie that never gets them is + * logged out after a while. The server does the same call and writes the new values into the cookie file, then checks + * that YouTube Music still sees it signed in. The result is shown on 관리자 → 수집. + * Only the server rotates (one holder per session); the collector CLI reads the file fresh. + */ +import { existsSync, readFileSync, statSync } from 'node:fs'; +import type { LogFn } from '../server'; +import { cookieForms, mergeCookies, parseSetCookies, writeCookieFile } from './cookie-jar'; +import { CookieError, YtMusic } from './ytmusic'; + +export interface CookieStatus { + /** ok: signed in · expired: Google ended the session (a new cookie is needed) · missing: no file · error: could not check (retried) */ + state: 'unknown' | 'ok' | 'expired' | 'missing' | 'error'; + message: string | null; + checkedAt: number | null; + /** Last time Google handed out a new session value. */ + rotatedAt: number | null; +} + +const ROTATE_URL = 'https://accounts.youtube.com/RotateCookies'; +const UA = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0 Safari/537.36'; + +export interface KeeperOptions { + fetch?: typeof fetch; + log?: LogFn; + /** Between rotations (the browser's 10 minutes). */ + everyMs?: number; + /** How often the file is looked at for a new cookie. */ + pollMs?: number; + /** Signed-in check; default: a YouTube Music song search that throws CookieError when logged out. */ + checkLogin?: (header: string) => Promise; +} + +export class CookieKeeper { + status: CookieStatus = { state: 'unknown', message: null, checkedAt: null, rotatedAt: null }; + private timer: ReturnType | null = null; + private running: Promise | null = null; + private lastRun = 0; + /** mtime of the file as of the last run (or our own write): a newer one is a cookie the admin put in. */ + private seenMtime = 0; + private fetch: typeof fetch; + private everyMs: number; + + constructor( + private path: string, + private opts: KeeperOptions = {}, + ) { + this.fetch = opts.fetch ?? fetch; + this.everyMs = opts.everyMs ?? 10 * 60_000; + } + + start(): void { + if (this.timer) return; + void this.tick(); + this.timer = setInterval(() => void this.tick(), this.opts.pollMs ?? 60_000); + } + + stop(): void { + if (this.timer) clearInterval(this.timer); + this.timer = null; + } + + private mtime(): number { + try { + return statSync(this.path).mtimeMs; + } catch { + return 0; + } + } + + /** Runs when the file changed, or every `everyMs` while the session is alive (an ended session waits for a new file). */ + async tick(now = Date.now()): Promise { + const changed = this.mtime() !== this.seenMtime; + const due = now - this.lastRun >= this.everyMs && this.status.state !== 'expired' && this.status.state !== 'missing'; + if (changed || due) await this.run(); + } + + run(): Promise { + this.running ??= this.rotate().finally(() => (this.running = null)); + return this.running; + } + + private set(state: CookieStatus['state'], message: string | null) { + if (state !== this.status.state) this.opts.log?.(state === 'ok' ? 'info' : 'warn', 'youtube cookie', { state, message }); + this.status = { ...this.status, state, message, checkedAt: Date.now() }; + } + + private async rotate(): Promise { + this.lastRun = Date.now(); + this.seenMtime = this.mtime(); + if (!existsSync(this.path)) return this.set('missing', `쿠키 파일이 없어요: ${this.path}`); + try { + const raw = readFileSync(this.path, 'utf8'); + const { header } = cookieForms(raw); + if (!/(?:^|; )SAPISID=/.test(header) || !/(?:^|; )__Secure-1PSID=/.test(header)) return this.set('expired', '쿠키에 로그인 값(SAPISID·__Secure-1PSID)이 없어요'); + const res = await this.fetch(ROTATE_URL, { + method: 'POST', + headers: { 'content-type': 'application/json', origin: 'https://www.youtube.com', cookie: header, 'user-agent': UA }, + body: `[1,"${-Math.floor(Math.random() * 9e18)}"]`, + signal: AbortSignal.timeout(15_000), + }); + if (res.status === 401 || res.status === 403) return this.set('expired', `구글이 이 로그인 세션을 끝냈어요(갱신 ${res.status})`); + if (!res.ok) return this.set('error', `쿠키 갱신 요청 실패(${res.status}) — 다음에 다시 시도해요`); + const updates = parseSetCookies(res.headers.getSetCookie()); + let current = header; + if (updates.length) { + // read again: the admin may have just put in a new cookie + const latest = readFileSync(this.path, 'utf8'); + if (latest === raw) { + writeCookieFile(this.path, mergeCookies(raw, updates)); + this.seenMtime = this.mtime(); + current = cookieForms(readFileSync(this.path, 'utf8')).header; + } + if (updates.some((u) => /PSIDTS$/.test(u.name))) this.status.rotatedAt = Date.now(); + } + await (this.opts.checkLogin ?? defaultCheck)(current); + this.set('ok', null); + } catch (e) { + if (e instanceof CookieError) return this.set('expired', e.message); + this.set('error', `확인하지 못했어요: ${(e as Error).message} — 다음에 다시 시도해요`); + } + } +} + +async function defaultCheck(header: string): Promise { + await new YtMusic(header).searchSongs('아이유 좋은 날'); +} diff --git a/apps/server/src/collector/ytmusic.ts b/apps/server/src/collector/ytmusic.ts index 98da222..cbd689d 100644 --- a/apps/server/src/collector/ytmusic.ts +++ b/apps/server/src/collector/ytmusic.ts @@ -2,6 +2,9 @@ * YouTube Music "노래" search with the user's cookie (docs/08 §1, docs/12 쿠키 — song search only). * Same approach as music_bot_v2: WEB_REMIX client + SAPISIDHASH. */ +import { statSync } from 'node:fs'; +import { cookieForms } from './cookie-jar'; + export interface YtmSong { videoId: string; title: string; @@ -50,18 +53,33 @@ export function parseSearch(json: unknown): YtmSong[] { } export class YtMusic { + private mtime = 0; + constructor( private cookie: string, private fetchImpl: typeof fetch = fetch, + /** The cookie file: read again whenever it changes, since the server keeps rotating its values (cookie-keeper.ts). */ + private path?: string, ) {} static async fromFile(path: string): Promise { - const cookie = (await Bun.file(path).text()).trim(); + const ytm = new YtMusic('', fetch, path); + await ytm.reload(); + return ytm; + } + + private async reload(): Promise { + if (!this.path) return; + const mtime = statSync(this.path).mtimeMs; + if (mtime === this.mtime) return; + const cookie = cookieForms(await Bun.file(this.path).text()).header; if (!/(?:^|; )SAPISID=/.test(cookie)) throw new CookieError('쿠키에 SAPISID가 없어요.'); - return new YtMusic(cookie); + this.cookie = cookie; + this.mtime = mtime; } async searchSongs(query: string): Promise { + await this.reload(); const sapisid = /(?:^|; )SAPISID=([^;]+)/.exec(this.cookie)![1]!; const ts = Math.floor(Date.now() / 1000); const hash = new Bun.CryptoHasher('sha1').update(`${ts} ${sapisid} ${ORIGIN}`).digest('hex'); diff --git a/apps/server/src/http/admin-catalog.ts b/apps/server/src/http/admin-catalog.ts index 514725b..e3e0841 100644 --- a/apps/server/src/http/admin-catalog.ts +++ b/apps/server/src/http/admin-catalog.ts @@ -301,7 +301,10 @@ export function mountCatalogAdmin( ) .all(); const cookie = d.db.query<{ log: string }, []>("SELECT log FROM collect_jobs WHERE kind = 'match-youtube' ORDER BY id DESC LIMIT 1").get(); - return c.json({ kinds: d.collect?.kinds ?? [], jobs, cookieProblem: !!cookie && /cookie|쿠키|로그인/i.test(cookie.log.split('\n').slice(-4).join('\n')) && /stopped/.test(cookie.log) }); + const status = d.cookieStatus?.() ?? null; + // without the keeper (dev), the last 원곡 찾기 stopping on the cookie is the only sign + const cookieProblem = status ? status.state === 'expired' || status.state === 'missing' : !!cookie && /cookie|쿠키|로그인/i.test(cookie.log.split('\n').slice(-4).join('\n')) && /stopped/.test(cookie.log); + return c.json({ kinds: d.collect?.kinds ?? [], jobs, cookieProblem, cookie: status }); }); app.get('/collect/:id', (c) => { const job = d.db.query, [number]>('SELECT * FROM collect_jobs WHERE id = ?').get(Number(c.req.param('id'))); diff --git a/apps/server/src/http/admin-quizzes.ts b/apps/server/src/http/admin-quizzes.ts index f740122..5711391 100644 --- a/apps/server/src/http/admin-quizzes.ts +++ b/apps/server/src/http/admin-quizzes.ts @@ -38,7 +38,7 @@ export interface QuizAdminDeps { let searcher: Promise<(q: string) => Promise> | null = null; function defaultSearch(q: string): Promise { // Picked once per process: with the cookie when it is logged in, else plain YouTube (Topic channels first). - searcher ??= pickSearcher(process.env.YT_COOKIE_PATH ?? '/run/secrets/ytm_cookie.txt', () => {}).then((s) => s.searchSongs); + searcher ??= pickSearcher(process.env.YT_COOKIE_PATH ?? '/run/secrets/music_quiz/ytm_cookie.txt', () => {}).then((s) => s.searchSongs); return searcher.then((f) => f(q)).catch((e) => { searcher = null; throw e; diff --git a/apps/server/src/http/admin.ts b/apps/server/src/http/admin.ts index d39deb8..7239569 100644 --- a/apps/server/src/http/admin.ts +++ b/apps/server/src/http/admin.ts @@ -12,6 +12,7 @@ import { checkEmbed, type EmbedState } from '../collector/oembed'; import { mountCatalogAdmin, songFilterFrom, type CollectControl } from './admin-catalog'; import { mountQuizAdmin } from './admin-quizzes'; import { CustomQuizzes } from '../quizzes/store'; +import type { CookieStatus } from '../collector/cookie-keeper'; import type { YtmSong } from '../collector/ytmusic'; import type { AudioLinks } from '../songs/audio'; @@ -35,6 +36,8 @@ export interface AdminDeps { adminClose(code: string): boolean; }; collect?: CollectControl; + /** 유튜브 뮤직 쿠키 상태 (collector/cookie-keeper.ts). */ + cookieStatus?: () => CookieStatus; /** 퀴즈 편집기의 유튜브 검색 (tests inject a fake). */ searchYoutube?: (q: string) => Promise; /** 플레이리스트 가져오기 (tests inject these; production reads LAVALINK_URL / LAVALINK_PW). */ diff --git a/apps/server/src/index.ts b/apps/server/src/index.ts index 9d2e203..eb4121f 100644 --- a/apps/server/src/index.ts +++ b/apps/server/src/index.ts @@ -1,8 +1,12 @@ +import { CookieKeeper } from './collector/cookie-keeper'; import { loadConfig } from './config'; import { jsonLog, startServer } from './server'; const config = loadConfig(); -const app = startServer({ config }); +// 유튜브 뮤직 쿠키: rotated like a browser tab does, so the copied session stays signed in (docs/08 §1) +const keeper = process.env.YT_COOKIE_PATH ? new CookieKeeper(process.env.YT_COOKIE_PATH, { log: jsonLog }) : null; +const app = startServer({ config, cookieStatus: keeper ? () => keeper.status : undefined }); +keeper?.start(); jsonLog('info', 'server started', { port: app.server.port, origin: config.publicOrigin, discord: config.discord !== null }); let stopping = false; @@ -10,6 +14,7 @@ const shutdown = async (signal: string) => { if (stopping) return; stopping = true; jsonLog('info', 'shutting down', { signal }); + keeper?.stop(); const force = setTimeout(() => process.exit(0), 10_000); await app.stop(); clearTimeout(force); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 44be36d..5fd962c 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -42,6 +42,8 @@ export interface StartOptions { /** Collector runner for the admin page (tests inject a fake). */ collect?: CollectControl; /** Quiz editor externals (YouTube search, Lavalink playlists); tests inject fakes. */ + /** 유튜브 뮤직 쿠키 상태 for 관리자 → 수집 (index.ts starts the keeper in production). */ + cookieStatus?: () => import('./collector/cookie-keeper').CookieStatus; quizAdmin?: Pick; } @@ -131,6 +133,7 @@ export function startServer(opts: StartOptions) { onGenresChange: loadGenres, rooms, collect, + cookieStatus: opts.cookieStatus, disconnectUser: (id) => gateway.disconnectUser(id), stats: () => ({ connections: gateway.connectionCount, diff --git a/apps/web/src/pages/admin/AdminMore.tsx b/apps/web/src/pages/admin/AdminMore.tsx index 89cdbe9..10a82ab 100644 --- a/apps/web/src/pages/admin/AdminMore.tsx +++ b/apps/web/src/pages/admin/AdminMore.tsx @@ -578,8 +578,37 @@ interface Job { } const STATUS: Record = { running: '진행 중', done: '완료', failed: '실패', stopped: '멈춤' }; +interface CookieStatus { + state: 'unknown' | 'ok' | 'expired' | 'missing' | 'error'; + message: string | null; + checkedAt: number | null; + rotatedAt: number | null; +} + +/** 유튜브 뮤직 쿠키: the server renews it every 10 minutes like a browser tab and checks it is still signed in. */ +function CookieLine({ s }: { s: CookieStatus }) { + const when = `마지막 확인 ${fmt(s.checkedAt)} · 마지막 갱신 ${fmt(s.rotatedAt)}`; + if (s.state === 'expired' || s.state === 'missing') + return ( +

+ ⚠ 유튜브 뮤직 쿠키 로그인이 풀렸어요{s.message ? ` (${s.message})` : ''}. 새 쿠키를 서버에 넣으면 1분 안에 다시 확인해요. 그전까지 원곡 찾기는 멈춰 있어요. +

+ ); + if (s.state === 'error') + return ( +

+ ⚠ 유튜브 뮤직 쿠키 상태를 이번에는 확인하지 못했어요{s.message ? ` (${s.message})` : ''}. {when} +

+ ); + return ( +

+ 유튜브 뮤직 쿠키: {s.state === 'ok' ? '로그인 유지 중 (10분마다 자동 갱신)' : '확인 중'} · {when} +

+ ); +} + export function CollectPage() { - const { data, reload } = useLoad<{ kinds: { id: string; label: string; limit?: boolean }[]; jobs: Job[]; cookieProblem: boolean }>('/api/admin/collect'); + const { data, reload } = useLoad<{ kinds: { id: string; label: string; limit?: boolean }[]; jobs: Job[]; cookieProblem: boolean; cookie: CookieStatus | null }>('/api/admin/collect'); const [limit, setLimit] = useState>({}); const [log, setLog] = useState<{ id: number; text: string } | null>(null); useEffect(() => { @@ -590,7 +619,7 @@ export function CollectPage() { return ( <>

수집

- {data?.cookieProblem &&

⚠ 유튜브 뮤직 쿠키가 로그아웃된 것 같아요. 새 쿠키를 서버에 넣어야 원곡 찾기가 다시 돌아요.

} + {data?.cookie ? : data?.cookieProblem &&

⚠ 유튜브 뮤직 쿠키가 로그아웃된 것 같아요. 새 쿠키를 서버에 넣어야 원곡 찾기가 다시 돌아요.

}

작업은 서버에서 따로 돌아가요(게임에는 영향 없음). 유튜브 원곡 찾기는 매일 08:10에 3,000곡씩 자동으로도 돌아요. 찾은 결과는 ‘원곡 찾기’ 화면에서 확인·수정할 수 있어요.

{data?.kinds.map((k) => { diff --git a/deploy/.env.example b/deploy/.env.example index f01bb71..caf21a6 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -5,4 +5,4 @@ SESSION_SECRET=change-me-at-least-32-characters-long DISCORD_CLIENT_ID=1561284577564037140 DISCORD_CLIENT_SECRET= SUPERADMIN_DISCORD_IDS= -YT_COOKIE_PATH=/run/secrets/ytm_cookie.txt +YT_COOKIE_PATH=/run/secrets/music_quiz/ytm_cookie.txt diff --git a/deploy/docker-compose.proxy.yml b/deploy/docker-compose.proxy.yml index 16eacf0..f75db4b 100644 --- a/deploy/docker-compose.proxy.yml +++ b/deploy/docker-compose.proxy.yml @@ -11,12 +11,15 @@ services: env_file: .env environment: TRUST_PROXY: "1" + # the cookie folder below (overrides an older YT_COOKIE_PATH in .env) + YT_COOKIE_PATH: /run/secrets/music_quiz/ytm_cookie.txt ports: - "${APP_PORT:-2974}:3000" volumes: - app-data:/data - # YouTube Music cookie for song search (collector / admin autofill, later stages). Read-only. - - ${YT_COOKIE_HOST_PATH:-/home/claude/.config/music_quiz/ytm_cookie.txt}:/run/secrets/ytm_cookie.txt:ro + # YouTube Music cookie (song search, 연령제한 음원). Writable: the server renews the cookie's session values in the file + # (collector/cookie-keeper.ts). The folder, not the file, so a cookie file replaced on the host is seen without a restart. + - ${YT_COOKIE_HOST_DIR:-/home/claude/.config/music_quiz}:/run/secrets/music_quiz mem_limit: 1g stop_grace_period: 15s logging: diff --git a/docs/08-youtube-playback.md b/docs/08-youtube-playback.md index a54645e..d921179 100644 --- a/docs/08-youtube-playback.md +++ b/docs/08-youtube-playback.md @@ -35,6 +35,14 @@ 5. 최고 점수 ≥ 80 → 자동 확정(`matched`), 50~79 → `candidate`(관리자 확인 대기), 그 아래 → `none` 6. 확정된 주소는 Lavalink `loadtracks`로 한 번 더 재생 가능 확인(지역 제한·삭제 영상 걸러냄) + **oEmbed로 퍼가기 허용 확인**(IFrame 재생이라 필수), 주 1회 재확인 +### 쿠키 갱신 (`collector/cookie-keeper.ts`, 2026-10-29) +- 원인: 구글 로그인 쿠키의 `__Secure-1PSIDTS`·`__Secure-3PSIDTS`·`*SIDCC`는 브라우저 탭이 10분마다 `accounts.youtube.com/RotateCookies`를 불러 새 값으로 바꾼다. 서버에 복사한 쿠키는 이 값을 못 받아 며칠 뒤 로그아웃됐다(10/19 쿠키 → 10/28 08:10 "로그인이 풀렸어요"). 쿠키를 뽑은 브라우저가 같은 세션을 계속 쓰면 더 빨리 끊긴다 +- 서버(index.ts, `YT_COOKIE_PATH`가 있을 때)가 10분마다 같은 요청(`POST accounts.youtube.com/RotateCookies`, origin `https://www.youtube.com`, 본문 `[1,"-숫자"]`)을 보내고 응답 `Set-Cookie`의 youtube.com 값을 쿠키 파일에 그 형식 그대로(헤더/cookies.txt) 덮어쓴다. 그다음 유튜브 뮤직 노래 검색으로 로그인 상태인지 확인 +- 갱신 401/403 또는 검색이 로그아웃 상태 → "로그인이 풀렸어요"(관리자 → 수집 맨 위). 새 쿠키를 넣을 때까지 갱신을 멈추고, 파일이 바뀌면 1분 안에 다시 확인. 네트워크 오류·429는 10분 뒤 재시도 +- 갱신은 서버 한 곳만(세션 하나에 갱신하는 쪽이 둘이면 서로의 값을 무효로 만든다). 수집기는 파일을 요청마다 새로 읽고(`YtMusic`은 파일이 바뀌면 다시 읽음, age-audio는 곡마다), yt-dlp가 돌려준 쿠키 중 값이 바뀐 것은 파일에 합친다 +- 잰 것(10/29, 이미 로그아웃된 쿠키): `accounts.youtube.com/RotateCookies` 200 + `SIDCC`만 새로 줌(PSIDTS 없음), 검색은 로그아웃 → 상태는 검색으로 판단해야 한다. 쿠키 없이 403, `accounts.google.com/RotateCookies`는 youtube 쿠키로 401 +- 쿠키 받는 법: 크롬 시크릿 창에서 로그인 → music.youtube.com → 쿠키 내보내기 → 로그아웃하지 말고 창을 닫는다(브라우저가 그 세션을 다시 쓰지 않게). 비밀번호 변경·"모든 기기 로그아웃"·구글 보안 확인이 있으면 다시 끊긴다 + ### 요청 속도 - 곡당 검색 1~3회, 2초 간격, 하루 최대 3,000곡. 차단 신호(HTTP 429, 봇 확인 페이지) 보이면 자동 중단 후 다음 날 재개 @@ -79,7 +87,7 @@ - 출제: 연령제한 곡은 음원을 받은 뒤에만 낸다(`songs/pool.ts`). 영상이 바뀌면 둘 다 다시 - 재생: `prepare`·`catchup`에 `videoId` 대신 `audio`(`/api/audio/<무작위 토큰>`, 6시간 유효, 문제마다 새로 발급)를 보낸다. 브라우저는 그 주소면 숨긴 `