diff --git a/src/github-ci.ts b/src/github-ci.ts index bef214f..52f2818 100644 --- a/src/github-ci.ts +++ b/src/github-ci.ts @@ -8,6 +8,7 @@ export interface GitHubCiMetadata { run_id: number; poll_count?: number; consecutive_errors?: number; + last_checked_at?: string; } interface GitHubActionsRunResponse { @@ -34,6 +35,12 @@ export interface GitHubRunCheckResult { completionMessage?: string; } +export const MAX_GITHUB_CONSECUTIVE_ERRORS = 5; +export const GITHUB_WATCH_BACKOFF_STEPS = [ + { afterMs: 60 * 60 * 1000, delayMs: 60_000 }, + { afterMs: 10 * 60 * 1000, delayMs: 30_000 }, +] as const; + function execGhApi(args: string[]): Promise { return new Promise((resolve, reject) => { execFile( @@ -87,6 +94,11 @@ export function parseGitHubCiMetadata( parsed.consecutive_errors! >= 0 ? parsed.consecutive_errors : undefined, + last_checked_at: + typeof parsed.last_checked_at === 'string' && + parsed.last_checked_at.trim() !== '' + ? parsed.last_checked_at + : undefined, }; } @@ -94,6 +106,28 @@ export function serializeGitHubCiMetadata(metadata: GitHubCiMetadata): string { return JSON.stringify(metadata); } +export function computeGitHubWatcherDelayMs( + task: Pick, + nowMs: number, +): number { + const baseDelayMs = Number.parseInt(task.schedule_value, 10); + const normalizedBaseDelayMs = + Number.isFinite(baseDelayMs) && baseDelayMs > 0 ? baseDelayMs : 15_000; + + const createdAtMs = new Date(task.created_at).getTime(); + const elapsedMs = Number.isFinite(createdAtMs) + ? Math.max(0, nowMs - createdAtMs) + : 0; + + for (const step of GITHUB_WATCH_BACKOFF_STEPS) { + if (elapsedMs >= step.afterMs) { + return Math.max(normalizedBaseDelayMs, step.delayMs); + } + } + + return normalizedBaseDelayMs; +} + function formatConclusionLabel(conclusion: string | null | undefined): string { switch (conclusion) { case 'success': diff --git a/src/task-scheduler.test.ts b/src/task-scheduler.test.ts index 385195f..c9480d2 100644 --- a/src/task-scheduler.test.ts +++ b/src/task-scheduler.test.ts @@ -106,6 +106,29 @@ vi.mock('./logger.js', () => ({ vi.mock('./github-ci.js', () => ({ checkGitHubActionsRun: checkGitHubActionsRunMock, + computeGitHubWatcherDelayMs: vi.fn((task: { schedule_value: string; created_at: string }, nowMs: number) => { + const baseDelayMs = Number.parseInt(task.schedule_value, 10); + const normalizedBaseDelayMs = + Number.isFinite(baseDelayMs) && baseDelayMs > 0 ? baseDelayMs : 15_000; + const createdAtMs = new Date(task.created_at).getTime(); + const elapsedMs = Number.isFinite(createdAtMs) + ? Math.max(0, nowMs - createdAtMs) + : 0; + + if (elapsedMs >= 60 * 60 * 1000) { + return Math.max(normalizedBaseDelayMs, 60_000); + } + if (elapsedMs >= 10 * 60 * 1000) { + return Math.max(normalizedBaseDelayMs, 30_000); + } + return normalizedBaseDelayMs; + }), + MAX_GITHUB_CONSECUTIVE_ERRORS: 5, + parseGitHubCiMetadata: vi.fn((raw: string | null | undefined) => { + if (!raw) return null; + return JSON.parse(raw); + }), + serializeGitHubCiMetadata: vi.fn((metadata: unknown) => JSON.stringify(metadata)), })); import { _initTestDatabase, createTask, getTaskById } from './db.js'; @@ -929,6 +952,8 @@ Managed by host-driven watcher. const task = getTaskById('task-github-running'); expect(task).toBeDefined(); expect(task?.next_run).not.toBe(dueAt); + expect(task?.ci_metadata).toContain('"poll_count":1'); + expect(task?.ci_metadata).toContain('"consecutive_errors":0'); }); it('sends a final message and deletes terminal GitHub watcher tasks', async () => { @@ -1003,6 +1028,146 @@ Managed by host-driven watcher. expect(getTaskById('task-github-complete')).toBeUndefined(); }); + it('backs off long-running GitHub watchers based on elapsed time', async () => { + const dueAt = new Date(Date.now() - 60_000).toISOString(); + createTask({ + id: 'task-github-backoff', + group_folder: 'shared-group', + chat_jid: 'shared@g.us', + agent_type: 'codex', + ci_provider: 'github', + ci_metadata: JSON.stringify({ + repo: 'owner/repo', + run_id: 222222, + poll_count: 9, + }), + prompt: ` +[BACKGROUND CI WATCH] + +Watch target: +GitHub Actions run 222222 + +Task ID: +task-github-backoff + +Check instructions: +Managed by host-driven watcher. + `.trim(), + schedule_type: 'interval', + schedule_value: '15000', + context_mode: 'isolated', + next_run: dueAt, + status: 'active', + created_at: new Date(Date.now() - 11 * 60_000).toISOString(), + }); + + const enqueueTask = vi.fn( + async (_groupJid: string, _taskId: string, fn: () => Promise) => { + await fn(); + }, + ); + + startSchedulerLoop({ + serviceAgentType: 'codex', + registeredGroups: () => ({ + 'shared@g.us': { + name: 'Shared', + folder: 'shared-group', + trigger: '@Codex', + added_at: '2026-02-22T00:00:00.000Z', + agentType: 'codex', + }, + }), + getSessions: () => ({}), + queue: { enqueueTask } as any, + onProcess: () => {}, + sendMessage: async () => {}, + }); + + await vi.advanceTimersByTimeAsync(10); + + const task = getTaskById('task-github-backoff'); + expect(task).toBeDefined(); + expect(new Date(task!.next_run!).getTime() - Date.now()).toBeGreaterThanOrEqual( + 29_000, + ); + expect(task?.ci_metadata).toContain('"poll_count":10'); + }); + + it('pauses GitHub watchers after repeated gh api failures', async () => { + checkGitHubActionsRunMock.mockRejectedValueOnce( + new Error('gh api failed: rate limit'), + ); + + const dueAt = new Date(Date.now() - 60_000).toISOString(); + createTask({ + id: 'task-github-pause', + group_folder: 'shared-group', + chat_jid: 'shared@g.us', + agent_type: 'codex', + ci_provider: 'github', + ci_metadata: JSON.stringify({ + repo: 'owner/repo', + run_id: 333333, + poll_count: 4, + consecutive_errors: 4, + }), + prompt: ` +[BACKGROUND CI WATCH] + +Watch target: +GitHub Actions run 333333 + +Task ID: +task-github-pause + +Check instructions: +Managed by host-driven watcher. + `.trim(), + schedule_type: 'interval', + schedule_value: '15000', + context_mode: 'isolated', + next_run: dueAt, + status: 'active', + created_at: '2026-02-22T00:00:00.000Z', + }); + + const sendMessage = vi.fn(async () => {}); + const enqueueTask = vi.fn( + async (_groupJid: string, _taskId: string, fn: () => Promise) => { + await fn(); + }, + ); + + startSchedulerLoop({ + serviceAgentType: 'codex', + registeredGroups: () => ({ + 'shared@g.us': { + name: 'Shared', + folder: 'shared-group', + trigger: '@Codex', + added_at: '2026-02-22T00:00:00.000Z', + agentType: 'codex', + }, + }), + getSessions: () => ({}), + queue: { enqueueTask } as any, + onProcess: () => {}, + sendMessage, + }); + + await vi.advanceTimersByTimeAsync(10); + + const task = getTaskById('task-github-pause'); + expect(task?.status).toBe('paused'); + expect(task?.ci_metadata).toContain('"consecutive_errors":5'); + expect(sendMessage).toHaveBeenCalledWith( + 'shared@g.us', + expect.stringContaining('gh api 연속 5회 실패'), + ); + expect(runAgentProcessMock).not.toHaveBeenCalled(); + }); + it('deletes active tasks that exceed max duration before they run', async () => { const enqueueTask = vi.fn(); createTask({ diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index fa51dfb..e114a4e 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -69,13 +69,20 @@ import { type StreamedOutputState, } from './streamed-output-evaluator.js'; import { + extractWatchCiTarget, getTaskQueueJid, getTaskRuntimeTaskId, isGitHubCiTask, shouldUseTaskScopedSession, } from './task-watch-status.js'; import { AgentType, RegisteredGroup, ScheduledTask } from './types.js'; -import { checkGitHubActionsRun } from './github-ci.js'; +import { + checkGitHubActionsRun, + computeGitHubWatcherDelayMs, + MAX_GITHUB_CONSECUTIVE_ERRORS, + parseGitHubCiMetadata, + serializeGitHubCiMetadata, +} from './github-ci.js'; export { extractWatchCiTarget, getTaskQueueJid, @@ -786,13 +793,23 @@ async function runGithubCiTask( deps: SchedulerDependencies, ): Promise { const startTime = Date.now(); + const runAtIso = new Date().toISOString(); let result: string | null = null; let error: string | null = null; let completedAndDeleted = false; + let paused = false; const statusTracker = createTaskStatusTracker(task, { sendTrackedMessage: deps.sendTrackedMessage, editTrackedMessage: deps.editTrackedMessage, }); + const parsedMetadata = parseGitHubCiMetadata(task.ci_metadata); + const metadata = parsedMetadata + ? { + ...parsedMetadata, + poll_count: (parsedMetadata.poll_count ?? 0) + 1, + last_checked_at: runAtIso, + } + : null; try { await statusTracker.update('checking'); @@ -800,6 +817,10 @@ async function runGithubCiTask( const check = await checkGitHubActionsRun(task); result = check.resultSummary; + if (metadata) { + metadata.consecutive_errors = 0; + } + if (check.terminal) { await statusTracker.update('completed'); if (check.completionMessage) { @@ -827,12 +848,19 @@ async function runGithubCiTask( } } catch (err) { error = getErrorMessage(err); + if (metadata) { + metadata.consecutive_errors = (metadata.consecutive_errors ?? 0) + 1; + } logger.error({ taskId: task.id, error }, 'GitHub CI watcher failed'); } const durationMs = Date.now() - startTime; const currentTask = getTaskById(task.id); - const nextRun = currentTask ? computeNextRun(task) : null; + const nextRun = currentTask + ? new Date( + Date.now() + computeGitHubWatcherDelayMs(currentTask, Date.now()), + ).toISOString() + : null; if (!currentTask) { if (!completedAndDeleted) { @@ -845,8 +873,32 @@ async function runGithubCiTask( return; } - if (error) { + if (metadata) { + updateTask(task.id, { ci_metadata: serializeGitHubCiMetadata(metadata) }); + } + + if ( + error && + metadata && + (metadata.consecutive_errors ?? 0) >= MAX_GITHUB_CONSECUTIVE_ERRORS + ) { + paused = true; + updateTask(task.id, { status: 'paused' }); + await deps.sendMessage( + task.chat_jid, + [ + `CI 감시 일시정지: ${extractWatchCiTarget(task.prompt) || task.id}`, + `- 사유: gh api 연속 ${metadata.consecutive_errors}회 실패`, + `- 마지막 오류: ${error.slice(0, 200)}`, + `- 태스크 ID: \`${task.id}\``, + ].join('\n'), + ); + } + + if (error && !paused) { await statusTracker.update('retrying', nextRun); + } else if (paused) { + // Paused tasks keep their current status message state; the pause notice is sent separately. } else if (nextRun) { await statusTracker.update('waiting', nextRun); } else {