145 lines
3.7 KiB
TypeScript
145 lines
3.7 KiB
TypeScript
import { TIMEZONE } from './config.js';
|
|
import { getRecentConsecutiveErrors, updateTask } from './db.js';
|
|
import { logger } from './logger.js';
|
|
import type { ScheduledTask } from './types.js';
|
|
|
|
const CONSECUTIVE_FAILURES_THRESHOLD = 3;
|
|
const DEFAULT_BACKOFF_STEPS_MS = [
|
|
3_600_000, // 1 hour
|
|
14_400_000, // 4 hours
|
|
43_200_000, // 12 hours
|
|
];
|
|
|
|
// Patterns that indicate a quota / auth / billing error (not transient).
|
|
const QUOTA_PATTERNS = [
|
|
/usage limit/i,
|
|
/rate limit/i,
|
|
/quota exceeded/i,
|
|
/billing/i,
|
|
/insufficient.*(credit|fund|balance)/i,
|
|
/exceeded.*plan/i,
|
|
/purchase more credits/i,
|
|
];
|
|
|
|
export interface SuspensionResult {
|
|
suspended: boolean;
|
|
suspendedUntil: string | null;
|
|
reason: string | null;
|
|
}
|
|
|
|
/**
|
|
* Detect if a quota/auth error message contains a human-readable
|
|
* retry-after date like "try again at Mar 26th, 2026 9:00 AM".
|
|
*/
|
|
export function parseRetryAfterDate(error: string): Date | null {
|
|
const match = error.match(
|
|
/try again (?:at|after)\s+(.+?\d{4}[\s,]+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i,
|
|
);
|
|
if (!match) return null;
|
|
|
|
const raw = match[1]
|
|
.replace(/(\d+)(?:st|nd|rd|th)/g, '$1')
|
|
.replace(/,\s*/g, ', ');
|
|
|
|
const parsed = new Date(raw);
|
|
if (Number.isNaN(parsed.getTime())) return null;
|
|
|
|
// Sanity: must be in the future and within 30 days
|
|
const now = Date.now();
|
|
if (parsed.getTime() <= now || parsed.getTime() > now + 30 * 86_400_000) {
|
|
return null;
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
function isQuotaError(error: string): boolean {
|
|
return QUOTA_PATTERNS.some((p) => p.test(error));
|
|
}
|
|
|
|
function computeBackoffMs(consecutiveCount: number): number {
|
|
const idx = Math.min(
|
|
consecutiveCount - CONSECUTIVE_FAILURES_THRESHOLD,
|
|
DEFAULT_BACKOFF_STEPS_MS.length - 1,
|
|
);
|
|
return DEFAULT_BACKOFF_STEPS_MS[Math.max(0, idx)];
|
|
}
|
|
|
|
/**
|
|
* Check recent run history and decide whether to suspend a task.
|
|
* Returns suspension info if the task should be suspended.
|
|
*/
|
|
export function evaluateTaskSuspension(
|
|
task: ScheduledTask,
|
|
latestError: string,
|
|
): SuspensionResult {
|
|
const noSuspension: SuspensionResult = {
|
|
suspended: false,
|
|
suspendedUntil: null,
|
|
reason: null,
|
|
};
|
|
|
|
if (!isQuotaError(latestError)) return noSuspension;
|
|
|
|
const recentErrors = getRecentConsecutiveErrors(
|
|
task.id,
|
|
CONSECUTIVE_FAILURES_THRESHOLD,
|
|
);
|
|
// +1 because the current error hasn't been logged yet
|
|
const totalConsecutive = recentErrors.length + 1;
|
|
|
|
if (totalConsecutive < CONSECUTIVE_FAILURES_THRESHOLD) return noSuspension;
|
|
|
|
// Try to parse an explicit retry-after date from the error
|
|
const retryDate = parseRetryAfterDate(latestError);
|
|
const suspendedUntil = retryDate
|
|
? retryDate.toISOString()
|
|
: new Date(Date.now() + computeBackoffMs(totalConsecutive)).toISOString();
|
|
|
|
return {
|
|
suspended: true,
|
|
suspendedUntil,
|
|
reason: latestError.slice(0, 200),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Apply suspension to a task in the DB.
|
|
*/
|
|
export function suspendTask(taskId: string, suspendedUntil: string): void {
|
|
updateTask(taskId, { suspended_until: suspendedUntil });
|
|
logger.info(
|
|
{ taskId, suspendedUntil },
|
|
'Task suspended due to repeated quota/auth errors',
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Format the suspension notification for Discord.
|
|
*/
|
|
const SUSPENSION_RESUME_FORMATTER = new Intl.DateTimeFormat('ko-KR', {
|
|
month: 'numeric',
|
|
day: 'numeric',
|
|
hour: '2-digit',
|
|
minute: '2-digit',
|
|
hour12: false,
|
|
timeZone: TIMEZONE,
|
|
});
|
|
|
|
export function formatSuspensionNotice(
|
|
task: ScheduledTask,
|
|
suspendedUntil: string,
|
|
reason: string,
|
|
): string {
|
|
const resumeLabel = SUSPENSION_RESUME_FORMATTER.format(
|
|
new Date(suspendedUntil),
|
|
);
|
|
|
|
const lines = [
|
|
`⏸ 태스크 일시 중단`,
|
|
`- 사유: ${reason}`,
|
|
`- 자동 재개: ${resumeLabel}`,
|
|
`- 태스크 ID: \`${task.id}\``,
|
|
];
|
|
return lines.join('\n');
|
|
}
|