claude-bot/EJClaw
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add GitHub watcher backoff and auto-pause safeguards

Browse Source
This commit is contained in:
Eyejoker
2026-03-25 22:43:35 +09:00
parent f51f282eb1
commit 09b4309d96
3 changed files with 254 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/github-ci.ts
View File

@@ -8,6 +8,7 @@ export interface GitHubCiMetadata {
run_id: number;
poll_count?: number;
consecutive_errors?: number;
last_checked_at?: string;
}
interface GitHubActionsRunResponse {
@@ -34,6 +35,12 @@ export interface GitHubRunCheckResult {
completionMessage?: string;
}
export const MAX_GITHUB_CONSECUTIVE_ERRORS = 5;
export const GITHUB_WATCH_BACKOFF_STEPS = [
{ afterMs: 60 * 60 * 1000, delayMs: 60_000 },
{ afterMs: 10 * 60 * 1000, delayMs: 30_000 },
] as const;
function execGhApi(args: string[]): Promise<string> {
return new Promise((resolve, reject) => {
execFile(
@@ -87,6 +94,11 @@ export function parseGitHubCiMetadata(
parsed.consecutive_errors! >= 0
? parsed.consecutive_errors
: undefined,
last_checked_at:
typeof parsed.last_checked_at === 'string' &&
parsed.last_checked_at.trim() !== ''
? parsed.last_checked_at
: undefined,
};
}
@@ -94,6 +106,28 @@ export function serializeGitHubCiMetadata(metadata: GitHubCiMetadata): string {
return JSON.stringify(metadata);
}
export function computeGitHubWatcherDelayMs(
task: Pick<ScheduledTask, 'schedule_value' | 'created_at'>,
nowMs: number,
): number {
const baseDelayMs = Number.parseInt(task.schedule_value, 10);
const normalizedBaseDelayMs =
Number.isFinite(baseDelayMs) && baseDelayMs > 0 ? baseDelayMs : 15_000;
const createdAtMs = new Date(task.created_at).getTime();
const elapsedMs = Number.isFinite(createdAtMs)
? Math.max(0, nowMs - createdAtMs)
: 0;
for (const step of GITHUB_WATCH_BACKOFF_STEPS) {
if (elapsedMs >= step.afterMs) {
return Math.max(normalizedBaseDelayMs, step.delayMs);
}
}
return normalizedBaseDelayMs;
}
function formatConclusionLabel(conclusion: string | null | undefined): string {
switch (conclusion) {
case 'success':

165
src/task-scheduler.test.ts
View File

@@ -106,6 +106,29 @@ vi.mock('./logger.js', () => ({
vi.mock('./github-ci.js', () => ({
checkGitHubActionsRun: checkGitHubActionsRunMock,
computeGitHubWatcherDelayMs: vi.fn((task: { schedule_value: string; created_at: string }, nowMs: number) => {
const baseDelayMs = Number.parseInt(task.schedule_value, 10);
const normalizedBaseDelayMs =
Number.isFinite(baseDelayMs) && baseDelayMs > 0 ? baseDelayMs : 15_000;
const createdAtMs = new Date(task.created_at).getTime();
const elapsedMs = Number.isFinite(createdAtMs)
? Math.max(0, nowMs - createdAtMs)
: 0;
if (elapsedMs >= 60 * 60 * 1000) {
return Math.max(normalizedBaseDelayMs, 60_000);
}
if (elapsedMs >= 10 * 60 * 1000) {
return Math.max(normalizedBaseDelayMs, 30_000);
}
return normalizedBaseDelayMs;
}),
MAX_GITHUB_CONSECUTIVE_ERRORS: 5,
parseGitHubCiMetadata: vi.fn((raw: string | null | undefined) => {
if (!raw) return null;
return JSON.parse(raw);
}),
serializeGitHubCiMetadata: vi.fn((metadata: unknown) => JSON.stringify(metadata)),
}));
import { _initTestDatabase, createTask, getTaskById } from './db.js';
@@ -929,6 +952,8 @@ Managed by host-driven watcher.
const task = getTaskById('task-github-running');
expect(task).toBeDefined();
expect(task?.next_run).not.toBe(dueAt);
expect(task?.ci_metadata).toContain('"poll_count":1');
expect(task?.ci_metadata).toContain('"consecutive_errors":0');
});
it('sends a final message and deletes terminal GitHub watcher tasks', async () => {
@@ -1003,6 +1028,146 @@ Managed by host-driven watcher.
expect(getTaskById('task-github-complete')).toBeUndefined();
});
it('backs off long-running GitHub watchers based on elapsed time', async () => {
const dueAt = new Date(Date.now() - 60_000).toISOString();
createTask({
id: 'task-github-backoff',
group_folder: 'shared-group',
chat_jid: 'shared@g.us',
agent_type: 'codex',
ci_provider: 'github',
ci_metadata: JSON.stringify({
repo: 'owner/repo',
run_id: 222222,
poll_count: 9,
}),
prompt: `
[BACKGROUND CI WATCH]
Watch target:
GitHub Actions run 222222
Task ID:
task-github-backoff
Check instructions:
Managed by host-driven watcher.
`.trim(),
schedule_type: 'interval',
schedule_value: '15000',
context_mode: 'isolated',
next_run: dueAt,
status: 'active',
created_at: new Date(Date.now() - 11 * 60_000).toISOString(),
});
const enqueueTask = vi.fn(
async (_groupJid: string, _taskId: string, fn: () => Promise<void>) => {
await fn();
},
);
startSchedulerLoop({
serviceAgentType: 'codex',
registeredGroups: () => ({
'shared@g.us': {
name: 'Shared',
folder: 'shared-group',
trigger: '@Codex',
added_at: '2026-02-22T00:00:00.000Z',
agentType: 'codex',
},
}),
getSessions: () => ({}),
queue: { enqueueTask } as any,
onProcess: () => {},
sendMessage: async () => {},
});
await vi.advanceTimersByTimeAsync(10);
const task = getTaskById('task-github-backoff');
expect(task).toBeDefined();
expect(new Date(task!.next_run!).getTime() - Date.now()).toBeGreaterThanOrEqual(
29_000,
);
expect(task?.ci_metadata).toContain('"poll_count":10');
});
it('pauses GitHub watchers after repeated gh api failures', async () => {
checkGitHubActionsRunMock.mockRejectedValueOnce(
new Error('gh api failed: rate limit'),
);
const dueAt = new Date(Date.now() - 60_000).toISOString();
createTask({
id: 'task-github-pause',
group_folder: 'shared-group',
chat_jid: 'shared@g.us',
agent_type: 'codex',
ci_provider: 'github',
ci_metadata: JSON.stringify({
repo: 'owner/repo',
run_id: 333333,
poll_count: 4,
consecutive_errors: 4,
}),
prompt: `
[BACKGROUND CI WATCH]
Watch target:
GitHub Actions run 333333
Task ID:
task-github-pause
Check instructions:
Managed by host-driven watcher.
`.trim(),
schedule_type: 'interval',
schedule_value: '15000',
context_mode: 'isolated',
next_run: dueAt,
status: 'active',
created_at: '2026-02-22T00:00:00.000Z',
});
const sendMessage = vi.fn(async () => {});
const enqueueTask = vi.fn(
async (_groupJid: string, _taskId: string, fn: () => Promise<void>) => {
await fn();
},
);
startSchedulerLoop({
serviceAgentType: 'codex',
registeredGroups: () => ({
'shared@g.us': {
name: 'Shared',
folder: 'shared-group',
trigger: '@Codex',
added_at: '2026-02-22T00:00:00.000Z',
agentType: 'codex',
},
}),
getSessions: () => ({}),
queue: { enqueueTask } as any,
onProcess: () => {},
sendMessage,
});
await vi.advanceTimersByTimeAsync(10);
const task = getTaskById('task-github-pause');
expect(task?.status).toBe('paused');
expect(task?.ci_metadata).toContain('"consecutive_errors":5');
expect(sendMessage).toHaveBeenCalledWith(
'shared@g.us',
expect.stringContaining('gh api 연속 5회 실패'),
);
expect(runAgentProcessMock).not.toHaveBeenCalled();
});
it('deletes active tasks that exceed max duration before they run', async () => {
const enqueueTask = vi.fn();
createTask({

58
src/task-scheduler.ts
View File

@@ -69,13 +69,20 @@ import {
type StreamedOutputState,
} from './streamed-output-evaluator.js';
import {
extractWatchCiTarget,
getTaskQueueJid,
getTaskRuntimeTaskId,
isGitHubCiTask,
shouldUseTaskScopedSession,
} from './task-watch-status.js';
import { AgentType, RegisteredGroup, ScheduledTask } from './types.js';
import { checkGitHubActionsRun } from './github-ci.js';
import {
checkGitHubActionsRun,
computeGitHubWatcherDelayMs,
MAX_GITHUB_CONSECUTIVE_ERRORS,
parseGitHubCiMetadata,
serializeGitHubCiMetadata,
} from './github-ci.js';
export {
extractWatchCiTarget,
getTaskQueueJid,
@@ -786,13 +793,23 @@ async function runGithubCiTask(
deps: SchedulerDependencies,
): Promise<void> {
const startTime = Date.now();
const runAtIso = new Date().toISOString();
let result: string | null = null;
let error: string | null = null;
let completedAndDeleted = false;
let paused = false;
const statusTracker = createTaskStatusTracker(task, {
sendTrackedMessage: deps.sendTrackedMessage,
editTrackedMessage: deps.editTrackedMessage,
});
const parsedMetadata = parseGitHubCiMetadata(task.ci_metadata);
const metadata = parsedMetadata
? {
...parsedMetadata,
poll_count: (parsedMetadata.poll_count ?? 0) + 1,
last_checked_at: runAtIso,
}
: null;
try {
await statusTracker.update('checking');
@@ -800,6 +817,10 @@ async function runGithubCiTask(
const check = await checkGitHubActionsRun(task);
result = check.resultSummary;
if (metadata) {
metadata.consecutive_errors = 0;
}
if (check.terminal) {
await statusTracker.update('completed');
if (check.completionMessage) {
@@ -827,12 +848,19 @@ async function runGithubCiTask(
}
} catch (err) {
error = getErrorMessage(err);
if (metadata) {
metadata.consecutive_errors = (metadata.consecutive_errors ?? 0) + 1;
}
logger.error({ taskId: task.id, error }, 'GitHub CI watcher failed');
}
const durationMs = Date.now() - startTime;
const currentTask = getTaskById(task.id);
const nextRun = currentTask ? computeNextRun(task) : null;
const nextRun = currentTask
? new Date(
Date.now() + computeGitHubWatcherDelayMs(currentTask, Date.now()),
).toISOString()
: null;
if (!currentTask) {
if (!completedAndDeleted) {
@@ -845,8 +873,32 @@ async function runGithubCiTask(
return;
}
if (error) {
if (metadata) {
updateTask(task.id, { ci_metadata: serializeGitHubCiMetadata(metadata) });
}
if (
error &&
metadata &&
(metadata.consecutive_errors ?? 0) >= MAX_GITHUB_CONSECUTIVE_ERRORS
) {
paused = true;
updateTask(task.id, { status: 'paused' });
await deps.sendMessage(
task.chat_jid,
[
`CI 감시 일시정지: ${extractWatchCiTarget(task.prompt) || task.id}`,
`- 사유: gh api 연속 ${metadata.consecutive_errors}회 실패`,
`- 마지막 오류: ${error.slice(0, 200)}`,
`- 태스크 ID: \`${task.id}\``,
].join('\n'),
);
}
if (error && !paused) {
await statusTracker.update('retrying', nextRun);
} else if (paused) {
// Paused tasks keep their current status message state; the pause notice is sent separately.
} else if (nextRun) {
await statusTracker.update('waiting', nextRun);
} else {