feat: add GitHub watcher backoff and auto-pause safeguards

src/github-ci.ts

@@ -8,6 +8,7 @@ export interface GitHubCiMetadata {
   run_id: number;
   poll_count?: number;
   consecutive_errors?: number;
+  last_checked_at?: string;
 }
 
 interface GitHubActionsRunResponse {
@@ -34,6 +35,12 @@ export interface GitHubRunCheckResult {
   completionMessage?: string;
 }
 
+export const MAX_GITHUB_CONSECUTIVE_ERRORS = 5;
+export const GITHUB_WATCH_BACKOFF_STEPS = [
+  { afterMs: 60 * 60 * 1000, delayMs: 60_000 },
+  { afterMs: 10 * 60 * 1000, delayMs: 30_000 },
+] as const;
+
 function execGhApi(args: string[]): Promise<string> {
   return new Promise((resolve, reject) => {
     execFile(
@@ -87,6 +94,11 @@ export function parseGitHubCiMetadata(
       parsed.consecutive_errors! >= 0
         ? parsed.consecutive_errors
         : undefined,
+    last_checked_at:
+      typeof parsed.last_checked_at === 'string' &&
+      parsed.last_checked_at.trim() !== ''
+        ? parsed.last_checked_at
+        : undefined,
   };
 }
 
@@ -94,6 +106,28 @@ export function serializeGitHubCiMetadata(metadata: GitHubCiMetadata): string {
   return JSON.stringify(metadata);
 }
 
+export function computeGitHubWatcherDelayMs(
+  task: Pick<ScheduledTask, 'schedule_value' | 'created_at'>,
+  nowMs: number,
+): number {
+  const baseDelayMs = Number.parseInt(task.schedule_value, 10);
+  const normalizedBaseDelayMs =
+    Number.isFinite(baseDelayMs) && baseDelayMs > 0 ? baseDelayMs : 15_000;
+
+  const createdAtMs = new Date(task.created_at).getTime();
+  const elapsedMs = Number.isFinite(createdAtMs)
+    ? Math.max(0, nowMs - createdAtMs)
+    : 0;
+
+  for (const step of GITHUB_WATCH_BACKOFF_STEPS) {
+    if (elapsedMs >= step.afterMs) {
+      return Math.max(normalizedBaseDelayMs, step.delayMs);
+    }
+  }
+
+  return normalizedBaseDelayMs;
+}
+
 function formatConclusionLabel(conclusion: string | null | undefined): string {
   switch (conclusion) {
     case 'success':

src/task-scheduler.test.ts

@@ -106,6 +106,29 @@ vi.mock('./logger.js', () => ({
 
 vi.mock('./github-ci.js', () => ({
   checkGitHubActionsRun: checkGitHubActionsRunMock,
+  computeGitHubWatcherDelayMs: vi.fn((task: { schedule_value: string; created_at: string }, nowMs: number) => {
+    const baseDelayMs = Number.parseInt(task.schedule_value, 10);
+    const normalizedBaseDelayMs =
+      Number.isFinite(baseDelayMs) && baseDelayMs > 0 ? baseDelayMs : 15_000;
+    const createdAtMs = new Date(task.created_at).getTime();
+    const elapsedMs = Number.isFinite(createdAtMs)
+      ? Math.max(0, nowMs - createdAtMs)
+      : 0;
+
+    if (elapsedMs >= 60 * 60 * 1000) {
+      return Math.max(normalizedBaseDelayMs, 60_000);
+    }
+    if (elapsedMs >= 10 * 60 * 1000) {
+      return Math.max(normalizedBaseDelayMs, 30_000);
+    }
+    return normalizedBaseDelayMs;
+  }),
+  MAX_GITHUB_CONSECUTIVE_ERRORS: 5,
+  parseGitHubCiMetadata: vi.fn((raw: string | null | undefined) => {
+    if (!raw) return null;
+    return JSON.parse(raw);
+  }),
+  serializeGitHubCiMetadata: vi.fn((metadata: unknown) => JSON.stringify(metadata)),
 }));
 
 import { _initTestDatabase, createTask, getTaskById } from './db.js';
@@ -929,6 +952,8 @@ Managed by host-driven watcher.
     const task = getTaskById('task-github-running');
     expect(task).toBeDefined();
     expect(task?.next_run).not.toBe(dueAt);
+    expect(task?.ci_metadata).toContain('"poll_count":1');
+    expect(task?.ci_metadata).toContain('"consecutive_errors":0');
   });
 
   it('sends a final message and deletes terminal GitHub watcher tasks', async () => {
@@ -1003,6 +1028,146 @@ Managed by host-driven watcher.
     expect(getTaskById('task-github-complete')).toBeUndefined();
   });
 
+  it('backs off long-running GitHub watchers based on elapsed time', async () => {
+    const dueAt = new Date(Date.now() - 60_000).toISOString();
+    createTask({
+      id: 'task-github-backoff',
+      group_folder: 'shared-group',
+      chat_jid: 'shared@g.us',
+      agent_type: 'codex',
+      ci_provider: 'github',
+      ci_metadata: JSON.stringify({
+        repo: 'owner/repo',
+        run_id: 222222,
+        poll_count: 9,
+      }),
+      prompt: `
+[BACKGROUND CI WATCH]
+
+Watch target:
+GitHub Actions run 222222
+
+Task ID:
+task-github-backoff
+
+Check instructions:
+Managed by host-driven watcher.
+      `.trim(),
+      schedule_type: 'interval',
+      schedule_value: '15000',
+      context_mode: 'isolated',
+      next_run: dueAt,
+      status: 'active',
+      created_at: new Date(Date.now() - 11 * 60_000).toISOString(),
+    });
+
+    const enqueueTask = vi.fn(
+      async (_groupJid: string, _taskId: string, fn: () => Promise<void>) => {
+        await fn();
+      },
+    );
+
+    startSchedulerLoop({
+      serviceAgentType: 'codex',
+      registeredGroups: () => ({
+        'shared@g.us': {
+          name: 'Shared',
+          folder: 'shared-group',
+          trigger: '@Codex',
+          added_at: '2026-02-22T00:00:00.000Z',
+          agentType: 'codex',
+        },
+      }),
+      getSessions: () => ({}),
+      queue: { enqueueTask } as any,
+      onProcess: () => {},
+      sendMessage: async () => {},
+    });
+
+    await vi.advanceTimersByTimeAsync(10);
+
+    const task = getTaskById('task-github-backoff');
+    expect(task).toBeDefined();
+    expect(new Date(task!.next_run!).getTime() - Date.now()).toBeGreaterThanOrEqual(
+      29_000,
+    );
+    expect(task?.ci_metadata).toContain('"poll_count":10');
+  });
+
+  it('pauses GitHub watchers after repeated gh api failures', async () => {
+    checkGitHubActionsRunMock.mockRejectedValueOnce(
+      new Error('gh api failed: rate limit'),
+    );
+
+    const dueAt = new Date(Date.now() - 60_000).toISOString();
+    createTask({
+      id: 'task-github-pause',
+      group_folder: 'shared-group',
+      chat_jid: 'shared@g.us',
+      agent_type: 'codex',
+      ci_provider: 'github',
+      ci_metadata: JSON.stringify({
+        repo: 'owner/repo',
+        run_id: 333333,
+        poll_count: 4,
+        consecutive_errors: 4,
+      }),
+      prompt: `
+[BACKGROUND CI WATCH]
+
+Watch target:
+GitHub Actions run 333333
+
+Task ID:
+task-github-pause
+
+Check instructions:
+Managed by host-driven watcher.
+      `.trim(),
+      schedule_type: 'interval',
+      schedule_value: '15000',
+      context_mode: 'isolated',
+      next_run: dueAt,
+      status: 'active',
+      created_at: '2026-02-22T00:00:00.000Z',
+    });
+
+    const sendMessage = vi.fn(async () => {});
+    const enqueueTask = vi.fn(
+      async (_groupJid: string, _taskId: string, fn: () => Promise<void>) => {
+        await fn();
+      },
+    );
+
+    startSchedulerLoop({
+      serviceAgentType: 'codex',
+      registeredGroups: () => ({
+        'shared@g.us': {
+          name: 'Shared',
+          folder: 'shared-group',
+          trigger: '@Codex',
+          added_at: '2026-02-22T00:00:00.000Z',
+          agentType: 'codex',
+        },
+      }),
+      getSessions: () => ({}),
+      queue: { enqueueTask } as any,
+      onProcess: () => {},
+      sendMessage,
+    });
+
+    await vi.advanceTimersByTimeAsync(10);
+
+    const task = getTaskById('task-github-pause');
+    expect(task?.status).toBe('paused');
+    expect(task?.ci_metadata).toContain('"consecutive_errors":5');
+    expect(sendMessage).toHaveBeenCalledWith(
+      'shared@g.us',
+      expect.stringContaining('gh api 연속 5회 실패'),
+    );
+    expect(runAgentProcessMock).not.toHaveBeenCalled();
+  });
+
   it('deletes active tasks that exceed max duration before they run', async () => {
     const enqueueTask = vi.fn();
     createTask({

src/task-scheduler.ts

@@ -69,13 +69,20 @@ import {
   type StreamedOutputState,
 } from './streamed-output-evaluator.js';
 import {
+  extractWatchCiTarget,
   getTaskQueueJid,
   getTaskRuntimeTaskId,
   isGitHubCiTask,
   shouldUseTaskScopedSession,
 } from './task-watch-status.js';
 import { AgentType, RegisteredGroup, ScheduledTask } from './types.js';
-import { checkGitHubActionsRun } from './github-ci.js';
+import {
+  checkGitHubActionsRun,
+  computeGitHubWatcherDelayMs,
+  MAX_GITHUB_CONSECUTIVE_ERRORS,
+  parseGitHubCiMetadata,
+  serializeGitHubCiMetadata,
+} from './github-ci.js';
 export {
   extractWatchCiTarget,
   getTaskQueueJid,
@@ -786,13 +793,23 @@ async function runGithubCiTask(
   deps: SchedulerDependencies,
 ): Promise<void> {
   const startTime = Date.now();
+  const runAtIso = new Date().toISOString();
   let result: string | null = null;
   let error: string | null = null;
   let completedAndDeleted = false;
+  let paused = false;
   const statusTracker = createTaskStatusTracker(task, {
     sendTrackedMessage: deps.sendTrackedMessage,
     editTrackedMessage: deps.editTrackedMessage,
   });
+  const parsedMetadata = parseGitHubCiMetadata(task.ci_metadata);
+  const metadata = parsedMetadata
+    ? {
+        ...parsedMetadata,
+        poll_count: (parsedMetadata.poll_count ?? 0) + 1,
+        last_checked_at: runAtIso,
+      }
+    : null;
 
   try {
     await statusTracker.update('checking');
@@ -800,6 +817,10 @@ async function runGithubCiTask(
     const check = await checkGitHubActionsRun(task);
     result = check.resultSummary;
 
+    if (metadata) {
+      metadata.consecutive_errors = 0;
+    }
+
     if (check.terminal) {
       await statusTracker.update('completed');
       if (check.completionMessage) {
@@ -827,12 +848,19 @@ async function runGithubCiTask(
     }
   } catch (err) {
     error = getErrorMessage(err);
+    if (metadata) {
+      metadata.consecutive_errors = (metadata.consecutive_errors ?? 0) + 1;
+    }
     logger.error({ taskId: task.id, error }, 'GitHub CI watcher failed');
   }
 
   const durationMs = Date.now() - startTime;
   const currentTask = getTaskById(task.id);
-  const nextRun = currentTask ? computeNextRun(task) : null;
+  const nextRun = currentTask
+    ? new Date(
+        Date.now() + computeGitHubWatcherDelayMs(currentTask, Date.now()),
+      ).toISOString()
+    : null;
 
   if (!currentTask) {
     if (!completedAndDeleted) {
@@ -845,8 +873,32 @@ async function runGithubCiTask(
     return;
   }
 
-  if (error) {
+  if (metadata) {
+    updateTask(task.id, { ci_metadata: serializeGitHubCiMetadata(metadata) });
+  }
+
+  if (
+    error &&
+    metadata &&
+    (metadata.consecutive_errors ?? 0) >= MAX_GITHUB_CONSECUTIVE_ERRORS
+  ) {
+    paused = true;
+    updateTask(task.id, { status: 'paused' });
+    await deps.sendMessage(
+      task.chat_jid,
+      [
+        `CI 감시 일시정지: ${extractWatchCiTarget(task.prompt) || task.id}`,
+        `- 사유: gh api 연속 ${metadata.consecutive_errors}회 실패`,
+        `- 마지막 오류: ${error.slice(0, 200)}`,
+        `- 태스크 ID: \`${task.id}\``,
+      ].join('\n'),
+    );
+  }
+
+  if (error && !paused) {
     await statusTracker.update('retrying', nextRun);
+  } else if (paused) {
+    // Paused tasks keep their current status message state; the pause notice is sent separately.
   } else if (nextRun) {
     await statusTracker.update('waiting', nextRun);
   } else {