refactor: extract shared utilities, protocol constants, and retry loop
Phase 3: provider-retry.ts — shared Claude rotation loop (SSOT) - retryClaudeWithRotation extracted from message-agent-executor + task-scheduler - ~200 lines of duplicated retry logic removed Phase 4: types.ts — AgentOutputPhase + VisiblePhase unified Phase 5: types.ts — AgentConfig extended with claudeThinking/claudeThinkingBudget Utilities (utils.ts): - getErrorMessage: 14 occurrences of instanceof Error pattern → 1 function - readJsonFile/writeJsonFile: 19 occurrences of JSON+fs pattern → 2 functions - fetchWithTimeout: 3 occurrences of AbortController pattern → 1 function - formatElapsedKorean: deduplicated from task-watch-status + message-turn-controller Protocol (agent-protocol.ts): - OUTPUT_START/END_MARKER centralized (runners keep local copies with SSOT reference) - IMAGE_TAG_RE, IPC constants documented Runner: show "대화 요약 중..." progress message during auto-compact Net: -137 lines, 354/354 tests passing
This commit is contained in:
@@ -32,6 +32,7 @@ interface ContainerInput {
|
|||||||
secrets?: Record<string, string>;
|
secrets?: Record<string, string>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Mirrors AgentOutput in src/agent-runner.ts (separate package, can't import directly). */
|
||||||
interface ContainerOutput {
|
interface ContainerOutput {
|
||||||
status: 'success' | 'error';
|
status: 'success' | 'error';
|
||||||
phase?: 'progress' | 'final' | 'tool-activity' | 'intermediate';
|
phase?: 'progress' | 'final' | 'tool-activity' | 'intermediate';
|
||||||
@@ -88,6 +89,7 @@ const IPC_INPUT_DIR = path.join(IPC_DIR, 'input');
|
|||||||
const IPC_INPUT_CLOSE_SENTINEL = path.join(IPC_INPUT_DIR, '_close');
|
const IPC_INPUT_CLOSE_SENTINEL = path.join(IPC_INPUT_DIR, '_close');
|
||||||
const IPC_POLL_MS = 500;
|
const IPC_POLL_MS = 500;
|
||||||
|
|
||||||
|
/** SSOT: src/agent-protocol.ts — keep in sync */
|
||||||
const IMAGE_TAG_RE = /\[Image:\s*(\/[^\]]+)\]/g;
|
const IMAGE_TAG_RE = /\[Image:\s*(\/[^\]]+)\]/g;
|
||||||
const MIME_TYPES: Record<string, string> = {
|
const MIME_TYPES: Record<string, string> = {
|
||||||
'.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
|
'.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
|
||||||
@@ -339,6 +341,14 @@ function createPreCompactHook(assistantName?: string): HookCallback {
|
|||||||
const preCompact = input as PreCompactHookInput;
|
const preCompact = input as PreCompactHookInput;
|
||||||
const transcriptPath = preCompact.transcript_path;
|
const transcriptPath = preCompact.transcript_path;
|
||||||
const sessionId = preCompact.session_id;
|
const sessionId = preCompact.session_id;
|
||||||
|
const trigger = preCompact.trigger || 'auto';
|
||||||
|
|
||||||
|
// Show compact status in chat so users know it's not just slow loading
|
||||||
|
writeOutput({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'progress',
|
||||||
|
result: trigger === 'auto' ? '대화 요약 중...' : '컴팩트 중...',
|
||||||
|
});
|
||||||
|
|
||||||
if (!transcriptPath || !fs.existsSync(transcriptPath)) {
|
if (!transcriptPath || !fs.existsSync(transcriptPath)) {
|
||||||
log('No transcript found for archiving');
|
log('No transcript found for archiving');
|
||||||
|
|||||||
@@ -50,6 +50,7 @@ const IPC_INPUT_DIR = path.join(IPC_DIR, 'input');
|
|||||||
const IPC_INPUT_CLOSE_SENTINEL = path.join(IPC_INPUT_DIR, '_close');
|
const IPC_INPUT_CLOSE_SENTINEL = path.join(IPC_INPUT_DIR, '_close');
|
||||||
const IPC_POLL_MS = 500;
|
const IPC_POLL_MS = 500;
|
||||||
|
|
||||||
|
/** SSOT: src/agent-protocol.ts — keep in sync */
|
||||||
const OUTPUT_START_MARKER = '---EJCLAW_OUTPUT_START---';
|
const OUTPUT_START_MARKER = '---EJCLAW_OUTPUT_START---';
|
||||||
const OUTPUT_END_MARKER = '---EJCLAW_OUTPUT_END---';
|
const OUTPUT_END_MARKER = '---EJCLAW_OUTPUT_END---';
|
||||||
|
|
||||||
@@ -134,6 +135,7 @@ function drainIpcInput(): string[] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function extractImagePaths(text: string): { cleanText: string; imagePaths: string[] } {
|
function extractImagePaths(text: string): { cleanText: string; imagePaths: string[] } {
|
||||||
|
/** SSOT: src/agent-protocol.ts — keep in sync */
|
||||||
const imagePattern = /\[Image:\s*(\/[^\]]+)\]/g;
|
const imagePattern = /\[Image:\s*(\/[^\]]+)\]/g;
|
||||||
const imagePaths: string[] = [];
|
const imagePaths: string[] = [];
|
||||||
let match: RegExpExecArray | null;
|
let match: RegExpExecArray | null;
|
||||||
|
|||||||
19
src/agent-protocol.ts
Normal file
19
src/agent-protocol.ts
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
/**
|
||||||
|
* Agent Protocol Constants (SSOT).
|
||||||
|
*
|
||||||
|
* Shared constants for host ↔ runner communication.
|
||||||
|
* Runners are separate packages and can't import this directly —
|
||||||
|
* they define local copies with comments referencing this file.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** Sentinel markers for robust stdout output parsing. */
|
||||||
|
export const OUTPUT_START_MARKER = '---EJCLAW_OUTPUT_START---';
|
||||||
|
export const OUTPUT_END_MARKER = '---EJCLAW_OUTPUT_END---';
|
||||||
|
|
||||||
|
/** Regex to extract [Image: /path/to/file] tags from agent text. */
|
||||||
|
export const IMAGE_TAG_RE = /\[Image:\s*(\/[^\]]+)\]/g;
|
||||||
|
|
||||||
|
/** IPC polling interval (ms) used by runners to check for follow-up messages. */
|
||||||
|
export const IPC_POLL_MS = 500;
|
||||||
|
export const IPC_INPUT_SUBDIR = 'input';
|
||||||
|
export const IPC_CLOSE_SENTINEL = '_close';
|
||||||
@@ -162,6 +162,14 @@ function prepareClaudeEnvironment(args: {
|
|||||||
if (args.group.agentConfig?.claudeEffort) {
|
if (args.group.agentConfig?.claudeEffort) {
|
||||||
args.env.CLAUDE_EFFORT = args.group.agentConfig.claudeEffort;
|
args.env.CLAUDE_EFFORT = args.group.agentConfig.claudeEffort;
|
||||||
}
|
}
|
||||||
|
if (args.group.agentConfig?.claudeThinking) {
|
||||||
|
args.env.CLAUDE_THINKING = args.group.agentConfig.claudeThinking;
|
||||||
|
}
|
||||||
|
if (args.group.agentConfig?.claudeThinkingBudget) {
|
||||||
|
args.env.CLAUDE_THINKING_BUDGET = String(
|
||||||
|
args.group.agentConfig.claudeThinkingBudget,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function prepareCodexSessionEnvironment(args: {
|
function prepareCodexSessionEnvironment(args: {
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import {
|
|||||||
resolveGroupIpcPath,
|
resolveGroupIpcPath,
|
||||||
resolveTaskRuntimeIpcPath,
|
resolveTaskRuntimeIpcPath,
|
||||||
} from './group-folder.js';
|
} from './group-folder.js';
|
||||||
|
import { writeJsonFile } from './utils.js';
|
||||||
|
|
||||||
export function writeTasksSnapshot(
|
export function writeTasksSnapshot(
|
||||||
groupFolder: string,
|
groupFolder: string,
|
||||||
@@ -28,7 +29,7 @@ export function writeTasksSnapshot(
|
|||||||
? tasks
|
? tasks
|
||||||
: tasks.filter((t) => t.groupFolder === groupFolder);
|
: tasks.filter((t) => t.groupFolder === groupFolder);
|
||||||
const tasksFile = path.join(groupIpcDir, 'current_tasks.json');
|
const tasksFile = path.join(groupIpcDir, 'current_tasks.json');
|
||||||
fs.writeFileSync(tasksFile, JSON.stringify(filteredTasks, null, 2));
|
writeJsonFile(tasksFile, filteredTasks, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface AvailableGroup {
|
export interface AvailableGroup {
|
||||||
@@ -48,12 +49,5 @@ export function writeGroupsSnapshot(
|
|||||||
fs.mkdirSync(groupIpcDir, { recursive: true });
|
fs.mkdirSync(groupIpcDir, { recursive: true });
|
||||||
const visibleGroups = isMain ? groups : [];
|
const visibleGroups = isMain ? groups : [];
|
||||||
const groupsFile = path.join(groupIpcDir, 'available_groups.json');
|
const groupsFile = path.join(groupIpcDir, 'available_groups.json');
|
||||||
fs.writeFileSync(
|
writeJsonFile(groupsFile, { groups: visibleGroups, lastSync: new Date().toISOString() }, true);
|
||||||
groupsFile,
|
|
||||||
JSON.stringify(
|
|
||||||
{ groups: visibleGroups, lastSync: new Date().toISOString() },
|
|
||||||
null,
|
|
||||||
2,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,9 +4,10 @@ import { PassThrough } from 'stream';
|
|||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import { spawn } from 'child_process';
|
import { spawn } from 'child_process';
|
||||||
|
|
||||||
// Sentinel markers must match agent-runner.ts
|
import {
|
||||||
const OUTPUT_START_MARKER = '---EJCLAW_OUTPUT_START---';
|
OUTPUT_START_MARKER,
|
||||||
const OUTPUT_END_MARKER = '---EJCLAW_OUTPUT_END---';
|
OUTPUT_END_MARKER,
|
||||||
|
} from './agent-protocol.js';
|
||||||
|
|
||||||
// Mock config
|
// Mock config
|
||||||
vi.mock('./config.js', () => ({
|
vi.mock('./config.js', () => ({
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import { ChildProcess, spawn } from 'child_process';
|
|||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
|
||||||
|
import { getErrorMessage } from './utils.js';
|
||||||
import {
|
import {
|
||||||
AGENT_MAX_OUTPUT_SIZE,
|
AGENT_MAX_OUTPUT_SIZE,
|
||||||
AGENT_TIMEOUT,
|
AGENT_TIMEOUT,
|
||||||
@@ -18,11 +19,8 @@ export {
|
|||||||
writeTasksSnapshot,
|
writeTasksSnapshot,
|
||||||
} from './agent-runner-snapshot.js';
|
} from './agent-runner-snapshot.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { AgentType, RegisteredGroup } from './types.js';
|
import { OUTPUT_END_MARKER, OUTPUT_START_MARKER } from './agent-protocol.js';
|
||||||
|
import { AgentOutputPhase, AgentType, RegisteredGroup } from './types.js';
|
||||||
// Sentinel markers for robust output parsing (must match agent-runner)
|
|
||||||
const OUTPUT_START_MARKER = '---EJCLAW_OUTPUT_START---';
|
|
||||||
const OUTPUT_END_MARKER = '---EJCLAW_OUTPUT_END---';
|
|
||||||
|
|
||||||
export interface AgentInput {
|
export interface AgentInput {
|
||||||
prompt: string;
|
prompt: string;
|
||||||
@@ -42,7 +40,7 @@ export interface AgentInput {
|
|||||||
export interface AgentOutput {
|
export interface AgentOutput {
|
||||||
status: 'success' | 'error';
|
status: 'success' | 'error';
|
||||||
result: string | null;
|
result: string | null;
|
||||||
phase?: 'progress' | 'final' | 'tool-activity' | 'intermediate';
|
phase?: AgentOutputPhase;
|
||||||
agentId?: string;
|
agentId?: string;
|
||||||
agentLabel?: string;
|
agentLabel?: string;
|
||||||
agentDone?: boolean;
|
agentDone?: boolean;
|
||||||
@@ -441,7 +439,7 @@ export async function runAgentProcess(
|
|||||||
resolve({
|
resolve({
|
||||||
status: 'error',
|
status: 'error',
|
||||||
result: null,
|
result: null,
|
||||||
error: `Failed to parse agent output: ${err instanceof Error ? err.message : String(err)}`,
|
error: `Failed to parse agent output: ${getErrorMessage(err)}`,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import path from 'path';
|
|||||||
import { DATA_DIR } from './config.js';
|
import { DATA_DIR } from './config.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { getCurrentToken, getAllTokens } from './token-rotation.js';
|
import { getCurrentToken, getAllTokens } from './token-rotation.js';
|
||||||
|
import { readJsonFile, writeJsonFile } from './utils.js';
|
||||||
|
|
||||||
const USAGE_CACHE_FILE = path.join(DATA_DIR, 'claude-usage-cache.json');
|
const USAGE_CACHE_FILE = path.join(DATA_DIR, 'claude-usage-cache.json');
|
||||||
|
|
||||||
@@ -54,18 +55,12 @@ let diskCacheLoaded = false;
|
|||||||
function loadUsageDiskCache(): void {
|
function loadUsageDiskCache(): void {
|
||||||
if (diskCacheLoaded) return;
|
if (diskCacheLoaded) return;
|
||||||
diskCacheLoaded = true;
|
diskCacheLoaded = true;
|
||||||
try {
|
usageDiskCache = readJsonFile<Record<string, UsageCacheEntry>>(USAGE_CACHE_FILE) ?? {};
|
||||||
if (fs.existsSync(USAGE_CACHE_FILE)) {
|
|
||||||
usageDiskCache = JSON.parse(fs.readFileSync(USAGE_CACHE_FILE, 'utf-8'));
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
/* start fresh */
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function saveUsageDiskCache(): void {
|
function saveUsageDiskCache(): void {
|
||||||
try {
|
try {
|
||||||
fs.writeFileSync(USAGE_CACHE_FILE, JSON.stringify(usageDiskCache));
|
writeJsonFile(USAGE_CACHE_FILE, usageDiskCache);
|
||||||
} catch {
|
} catch {
|
||||||
/* best effort */
|
/* best effort */
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ import {
|
|||||||
findNextAvailable,
|
findNextAvailable,
|
||||||
parseRetryAfterFromError,
|
parseRetryAfterFromError,
|
||||||
} from './token-rotation-base.js';
|
} from './token-rotation-base.js';
|
||||||
|
import { readJsonFile, writeJsonFile } from './utils.js';
|
||||||
|
|
||||||
const STATE_FILE = path.join(DATA_DIR, 'codex-rotation-state.json');
|
const STATE_FILE = path.join(DATA_DIR, 'codex-rotation-state.json');
|
||||||
|
|
||||||
@@ -93,22 +94,22 @@ export function initCodexTokenRotation(): void {
|
|||||||
const authPath = path.join(ACCOUNTS_DIR, dir, 'auth.json');
|
const authPath = path.join(ACCOUNTS_DIR, dir, 'auth.json');
|
||||||
if (!fs.existsSync(authPath)) continue;
|
if (!fs.existsSync(authPath)) continue;
|
||||||
|
|
||||||
try {
|
const data = readJsonFile<{ tokens?: { account_id?: string; id_token?: string } }>(authPath);
|
||||||
const data = JSON.parse(fs.readFileSync(authPath, 'utf-8'));
|
if (!data) {
|
||||||
const accountId = data?.tokens?.account_id || `account-${dir}`;
|
|
||||||
const jwt = parseJwtAuth(data?.tokens?.id_token || '');
|
|
||||||
const planType = jwt.planType;
|
|
||||||
accounts.push({
|
|
||||||
index: accounts.length,
|
|
||||||
authPath,
|
|
||||||
accountId,
|
|
||||||
planType,
|
|
||||||
subscriptionUntil: jwt.expiresAt,
|
|
||||||
rateLimitedUntil: null,
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
logger.warn({ authPath }, 'Failed to parse codex account auth.json');
|
logger.warn({ authPath }, 'Failed to parse codex account auth.json');
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
|
const accountId = data?.tokens?.account_id || `account-${dir}`;
|
||||||
|
const jwt = parseJwtAuth(data?.tokens?.id_token || '');
|
||||||
|
const planType = jwt.planType;
|
||||||
|
accounts.push({
|
||||||
|
index: accounts.length,
|
||||||
|
authPath,
|
||||||
|
accountId,
|
||||||
|
planType,
|
||||||
|
subscriptionUntil: jwt.expiresAt,
|
||||||
|
rateLimitedUntil: null,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if (accounts.length > 1) loadCodexState();
|
if (accounts.length > 1) loadCodexState();
|
||||||
@@ -128,80 +129,84 @@ function saveCodexState(): void {
|
|||||||
resetAts: accounts.map((a) => a.resetAt ?? null),
|
resetAts: accounts.map((a) => a.resetAt ?? null),
|
||||||
resetD7Ats: accounts.map((a) => a.resetD7At ?? null),
|
resetD7Ats: accounts.map((a) => a.resetD7At ?? null),
|
||||||
};
|
};
|
||||||
fs.writeFileSync(STATE_FILE, JSON.stringify(state));
|
writeJsonFile(STATE_FILE, state);
|
||||||
} catch {
|
} catch {
|
||||||
/* best effort */
|
/* best effort */
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function loadCodexState(): void {
|
function loadCodexState(): void {
|
||||||
try {
|
const state = readJsonFile<{
|
||||||
if (!fs.existsSync(STATE_FILE)) return;
|
currentIndex?: number;
|
||||||
const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
|
rateLimits?: (number | null)[];
|
||||||
const now = Date.now();
|
usagePcts?: (number | null)[];
|
||||||
if (
|
usageD7Pcts?: (number | null)[];
|
||||||
typeof state.currentIndex === 'number' &&
|
resetAts?: (string | null)[];
|
||||||
state.currentIndex < accounts.length
|
resetD7Ats?: (string | null)[];
|
||||||
) {
|
}>(STATE_FILE);
|
||||||
currentIndex = state.currentIndex;
|
if (!state) return;
|
||||||
}
|
|
||||||
if (Array.isArray(state.rateLimits)) {
|
const now = Date.now();
|
||||||
for (
|
if (
|
||||||
let i = 0;
|
typeof state.currentIndex === 'number' &&
|
||||||
i < Math.min(state.rateLimits.length, accounts.length);
|
state.currentIndex < accounts.length
|
||||||
i++
|
) {
|
||||||
) {
|
currentIndex = state.currentIndex;
|
||||||
const until = state.rateLimits[i];
|
|
||||||
if (typeof until === 'number' && until > now) {
|
|
||||||
accounts[i].rateLimitedUntil = until;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (Array.isArray(state.usagePcts)) {
|
|
||||||
for (
|
|
||||||
let i = 0;
|
|
||||||
i < Math.min(state.usagePcts.length, accounts.length);
|
|
||||||
i++
|
|
||||||
) {
|
|
||||||
if (typeof state.usagePcts[i] === 'number')
|
|
||||||
accounts[i].lastUsagePct = state.usagePcts[i];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (Array.isArray(state.usageD7Pcts)) {
|
|
||||||
for (
|
|
||||||
let i = 0;
|
|
||||||
i < Math.min(state.usageD7Pcts.length, accounts.length);
|
|
||||||
i++
|
|
||||||
) {
|
|
||||||
if (typeof state.usageD7Pcts[i] === 'number')
|
|
||||||
accounts[i].lastUsageD7Pct = state.usageD7Pcts[i];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (Array.isArray(state.resetAts)) {
|
|
||||||
for (
|
|
||||||
let i = 0;
|
|
||||||
i < Math.min(state.resetAts.length, accounts.length);
|
|
||||||
i++
|
|
||||||
) {
|
|
||||||
if (state.resetAts[i]) accounts[i].resetAt = state.resetAts[i];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (Array.isArray(state.resetD7Ats)) {
|
|
||||||
for (
|
|
||||||
let i = 0;
|
|
||||||
i < Math.min(state.resetD7Ats.length, accounts.length);
|
|
||||||
i++
|
|
||||||
) {
|
|
||||||
if (state.resetD7Ats[i]) accounts[i].resetD7At = state.resetD7Ats[i];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
logger.info(
|
|
||||||
{ currentIndex, accountCount: accounts.length },
|
|
||||||
'Codex rotation state restored',
|
|
||||||
);
|
|
||||||
} catch {
|
|
||||||
/* start fresh */
|
|
||||||
}
|
}
|
||||||
|
if (Array.isArray(state.rateLimits)) {
|
||||||
|
for (
|
||||||
|
let i = 0;
|
||||||
|
i < Math.min(state.rateLimits.length, accounts.length);
|
||||||
|
i++
|
||||||
|
) {
|
||||||
|
const until = state.rateLimits[i];
|
||||||
|
if (typeof until === 'number' && until > now) {
|
||||||
|
accounts[i].rateLimitedUntil = until;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (Array.isArray(state.usagePcts)) {
|
||||||
|
for (
|
||||||
|
let i = 0;
|
||||||
|
i < Math.min(state.usagePcts.length, accounts.length);
|
||||||
|
i++
|
||||||
|
) {
|
||||||
|
if (typeof state.usagePcts[i] === 'number')
|
||||||
|
accounts[i].lastUsagePct = state.usagePcts[i]!;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (Array.isArray(state.usageD7Pcts)) {
|
||||||
|
for (
|
||||||
|
let i = 0;
|
||||||
|
i < Math.min(state.usageD7Pcts.length, accounts.length);
|
||||||
|
i++
|
||||||
|
) {
|
||||||
|
if (typeof state.usageD7Pcts[i] === 'number')
|
||||||
|
accounts[i].lastUsageD7Pct = state.usageD7Pcts[i]!;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (Array.isArray(state.resetAts)) {
|
||||||
|
for (
|
||||||
|
let i = 0;
|
||||||
|
i < Math.min(state.resetAts.length, accounts.length);
|
||||||
|
i++
|
||||||
|
) {
|
||||||
|
if (state.resetAts[i]) accounts[i].resetAt = state.resetAts[i]!;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (Array.isArray(state.resetD7Ats)) {
|
||||||
|
for (
|
||||||
|
let i = 0;
|
||||||
|
i < Math.min(state.resetD7Ats.length, accounts.length);
|
||||||
|
i++
|
||||||
|
) {
|
||||||
|
if (state.resetD7Ats[i]) accounts[i].resetD7At = state.resetD7Ats[i]!;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
logger.info(
|
||||||
|
{ currentIndex, accountCount: accounts.length },
|
||||||
|
'Codex rotation state restored',
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Get the auth.json path for the current active account. */
|
/** Get the auth.json path for the current active account. */
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import {
|
|||||||
ScheduledTask,
|
ScheduledTask,
|
||||||
TaskRunLog,
|
TaskRunLog,
|
||||||
} from './types.js';
|
} from './types.js';
|
||||||
|
import { readJsonFile } from './utils.js';
|
||||||
|
|
||||||
let db: Database.Database;
|
let db: Database.Database;
|
||||||
|
|
||||||
@@ -1300,14 +1301,14 @@ export function isPairedRoomJid(jid: string): boolean {
|
|||||||
function migrateJsonState(): void {
|
function migrateJsonState(): void {
|
||||||
const migrateFile = (filename: string) => {
|
const migrateFile = (filename: string) => {
|
||||||
const filePath = path.join(DATA_DIR, filename);
|
const filePath = path.join(DATA_DIR, filename);
|
||||||
if (!fs.existsSync(filePath)) return null;
|
const data = readJsonFile(filePath);
|
||||||
|
if (data === null) return null;
|
||||||
try {
|
try {
|
||||||
const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
|
|
||||||
fs.renameSync(filePath, `${filePath}.migrated`);
|
fs.renameSync(filePath, `${filePath}.migrated`);
|
||||||
return data;
|
|
||||||
} catch {
|
} catch {
|
||||||
return null;
|
/* best effort */
|
||||||
}
|
}
|
||||||
|
return data;
|
||||||
};
|
};
|
||||||
|
|
||||||
// Migrate router_state.json
|
// Migrate router_state.json
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
|
||||||
|
import { writeJsonFile } from './utils.js';
|
||||||
|
|
||||||
function resolveInputDir(ipcDir: string): string {
|
function resolveInputDir(ipcDir: string): string {
|
||||||
return path.join(ipcDir, 'input');
|
return path.join(ipcDir, 'input');
|
||||||
}
|
}
|
||||||
@@ -11,7 +13,7 @@ export function queueFollowUpMessage(ipcDir: string, text: string): string {
|
|||||||
const filename = `${Date.now()}-${Math.random().toString(36).slice(2, 6)}.json`;
|
const filename = `${Date.now()}-${Math.random().toString(36).slice(2, 6)}.json`;
|
||||||
const filepath = path.join(inputDir, filename);
|
const filepath = path.join(inputDir, filename);
|
||||||
const tempPath = `${filepath}.tmp`;
|
const tempPath = `${filepath}.tmp`;
|
||||||
fs.writeFileSync(tempPath, JSON.stringify({ type: 'message', text }));
|
writeJsonFile(tempPath, { type: 'message', text });
|
||||||
fs.renameSync(tempPath, filepath);
|
fs.renameSync(tempPath, filepath);
|
||||||
return filename;
|
return filename;
|
||||||
}
|
}
|
||||||
|
|||||||
20
src/ipc.ts
20
src/ipc.ts
@@ -9,6 +9,7 @@ import {
|
|||||||
SERVICE_AGENT_TYPE,
|
SERVICE_AGENT_TYPE,
|
||||||
TIMEZONE,
|
TIMEZONE,
|
||||||
} from './config.js';
|
} from './config.js';
|
||||||
|
import { readJsonFile } from './utils.js';
|
||||||
import { AvailableGroup } from './agent-runner.js';
|
import { AvailableGroup } from './agent-runner.js';
|
||||||
import { createTask, deleteTask, getTaskById, updateTask } from './db.js';
|
import { createTask, deleteTask, getTaskById, updateTask } from './db.js';
|
||||||
import { isValidGroupFolder } from './group-folder.js';
|
import { isValidGroupFolder } from './group-folder.js';
|
||||||
@@ -78,22 +79,24 @@ export function startIpcWatcher(deps: IpcDeps): void {
|
|||||||
for (const file of messageFiles) {
|
for (const file of messageFiles) {
|
||||||
const filePath = path.join(messagesDir, file);
|
const filePath = path.join(messagesDir, file);
|
||||||
try {
|
try {
|
||||||
const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
|
const data = readJsonFile(filePath);
|
||||||
if (data.type === 'message' && data.chatJid && data.text) {
|
if (!data || typeof data !== 'object') throw new Error('Invalid JSON');
|
||||||
|
const msg = data as { type?: string; chatJid?: string; text?: string };
|
||||||
|
if (msg.type === 'message' && msg.chatJid && msg.text) {
|
||||||
// Authorization: verify this group can send to this chatJid
|
// Authorization: verify this group can send to this chatJid
|
||||||
const targetGroup = registeredGroups[data.chatJid];
|
const targetGroup = registeredGroups[msg.chatJid];
|
||||||
if (
|
if (
|
||||||
isMain ||
|
isMain ||
|
||||||
(targetGroup && targetGroup.folder === sourceGroup)
|
(targetGroup && targetGroup.folder === sourceGroup)
|
||||||
) {
|
) {
|
||||||
await deps.sendMessage(data.chatJid, data.text);
|
await deps.sendMessage(msg.chatJid, msg.text);
|
||||||
logger.info(
|
logger.info(
|
||||||
{ chatJid: data.chatJid, sourceGroup },
|
{ chatJid: msg.chatJid, sourceGroup },
|
||||||
'IPC message sent',
|
'IPC message sent',
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{ chatJid: data.chatJid, sourceGroup },
|
{ chatJid: msg.chatJid, sourceGroup },
|
||||||
'Unauthorized IPC message attempt blocked',
|
'Unauthorized IPC message attempt blocked',
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -129,9 +132,10 @@ export function startIpcWatcher(deps: IpcDeps): void {
|
|||||||
for (const file of taskFiles) {
|
for (const file of taskFiles) {
|
||||||
const filePath = path.join(tasksDir, file);
|
const filePath = path.join(tasksDir, file);
|
||||||
try {
|
try {
|
||||||
const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
|
const data = readJsonFile(filePath);
|
||||||
|
if (!data || typeof data !== 'object') throw new Error('Invalid JSON');
|
||||||
// Pass source group identity to processTaskIpc for authorization
|
// Pass source group identity to processTaskIpc for authorization
|
||||||
await processTaskIpc(data, sourceGroup, isMain, deps);
|
await processTaskIpc(data as Parameters<typeof processTaskIpc>[0], sourceGroup, isMain, deps);
|
||||||
fs.unlinkSync(filePath);
|
fs.unlinkSync(filePath);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.error(
|
logger.error(
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
import { getErrorMessage } from './utils.js';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
AgentOutput,
|
AgentOutput,
|
||||||
@@ -16,7 +17,6 @@ import {
|
|||||||
isClaudeAuthError,
|
isClaudeAuthError,
|
||||||
isClaudeAuthExpiredMessage,
|
isClaudeAuthExpiredMessage,
|
||||||
isClaudeUsageExhaustedMessage,
|
isClaudeUsageExhaustedMessage,
|
||||||
shouldRotateClaudeToken,
|
|
||||||
} from './agent-error-detection.js';
|
} from './agent-error-detection.js';
|
||||||
import {
|
import {
|
||||||
detectFallbackTrigger,
|
detectFallbackTrigger,
|
||||||
@@ -28,6 +28,7 @@ import {
|
|||||||
isUsageExhausted,
|
isUsageExhausted,
|
||||||
markPrimaryCooldown,
|
markPrimaryCooldown,
|
||||||
} from './provider-fallback.js';
|
} from './provider-fallback.js';
|
||||||
|
import { runClaudeRotationLoop } from './provider-retry.js';
|
||||||
import { shouldResetSessionOnAgentFailure } from './session-recovery.js';
|
import { shouldResetSessionOnAgentFailure } from './session-recovery.js';
|
||||||
import {
|
import {
|
||||||
detectCodexRotationTrigger,
|
detectCodexRotationTrigger,
|
||||||
@@ -35,11 +36,7 @@ import {
|
|||||||
getCodexAccountCount,
|
getCodexAccountCount,
|
||||||
markCodexTokenHealthy,
|
markCodexTokenHealthy,
|
||||||
} from './codex-token-rotation.js';
|
} from './codex-token-rotation.js';
|
||||||
import {
|
import { getTokenCount } from './token-rotation.js';
|
||||||
rotateToken,
|
|
||||||
getTokenCount,
|
|
||||||
markTokenHealthy,
|
|
||||||
} from './token-rotation.js';
|
|
||||||
import type { RegisteredGroup } from './types.js';
|
import type { RegisteredGroup } from './types.js';
|
||||||
|
|
||||||
export interface MessageAgentExecutorDeps {
|
export interface MessageAgentExecutorDeps {
|
||||||
@@ -392,10 +389,7 @@ export async function runAgentForGroup(
|
|||||||
const retryAttempt = await runAttempt('codex');
|
const retryAttempt = await runAttempt('codex');
|
||||||
|
|
||||||
if (retryAttempt.error) {
|
if (retryAttempt.error) {
|
||||||
const errMsg =
|
const errMsg = getErrorMessage(retryAttempt.error);
|
||||||
retryAttempt.error instanceof Error
|
|
||||||
? retryAttempt.error.message
|
|
||||||
: String(retryAttempt.error);
|
|
||||||
const retryTrigger = detectCodexRotationTrigger(errMsg);
|
const retryTrigger = detectCodexRotationTrigger(errMsg);
|
||||||
if (retryTrigger.shouldRotate) {
|
if (retryTrigger.shouldRotate) {
|
||||||
trigger = { reason: retryTrigger.reason };
|
trigger = { reason: retryTrigger.reason };
|
||||||
@@ -486,156 +480,54 @@ export async function runAgentForGroup(
|
|||||||
},
|
},
|
||||||
rotationMessage?: string,
|
rotationMessage?: string,
|
||||||
): Promise<'success' | 'error'> => {
|
): Promise<'success' | 'error'> => {
|
||||||
let trigger = initialTrigger;
|
const logCtx = {
|
||||||
let lastRotationMessage = rotationMessage;
|
chatJid,
|
||||||
|
group: group.name,
|
||||||
|
groupFolder: group.folder,
|
||||||
|
runId,
|
||||||
|
};
|
||||||
|
|
||||||
while (
|
const outcome = await runClaudeRotationLoop(
|
||||||
shouldRotateClaudeToken(trigger.reason) &&
|
initialTrigger,
|
||||||
getTokenCount() > 1 &&
|
async () => {
|
||||||
rotateToken(lastRotationMessage, { ignoreRateLimits: true })
|
const attempt = await runAttempt('claude');
|
||||||
) {
|
return {
|
||||||
logger.info(
|
output: attempt.output,
|
||||||
{ chatJid, group: group.name, runId, reason: trigger.reason },
|
thrownError: attempt.error,
|
||||||
'Claude rate-limited, retrying with rotated account',
|
sawOutput: attempt.sawOutput,
|
||||||
);
|
sawSuccessNullResult: attempt.sawSuccessNullResultWithoutOutput,
|
||||||
|
streamedTriggerReason: attempt.streamedTriggerReason,
|
||||||
const retryAttempt = await runAttempt('claude');
|
|
||||||
|
|
||||||
if (retryAttempt.error) {
|
|
||||||
if (!retryAttempt.sawOutput) {
|
|
||||||
const errMsg =
|
|
||||||
retryAttempt.error instanceof Error
|
|
||||||
? retryAttempt.error.message
|
|
||||||
: String(retryAttempt.error);
|
|
||||||
const retryTrigger = retryAttempt.streamedTriggerReason
|
|
||||||
? {
|
|
||||||
shouldFallback: true,
|
|
||||||
reason: retryAttempt.streamedTriggerReason.reason,
|
|
||||||
retryAfterMs: retryAttempt.streamedTriggerReason.retryAfterMs,
|
|
||||||
}
|
|
||||||
: detectFallbackTrigger(errMsg);
|
|
||||||
if (retryTrigger.shouldFallback) {
|
|
||||||
trigger = {
|
|
||||||
reason: retryTrigger.reason,
|
|
||||||
retryAfterMs: retryTrigger.retryAfterMs,
|
|
||||||
};
|
|
||||||
lastRotationMessage = errMsg;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.error(
|
|
||||||
{
|
|
||||||
chatJid,
|
|
||||||
group: group.name,
|
|
||||||
groupFolder: group.folder,
|
|
||||||
runId,
|
|
||||||
provider: 'claude',
|
|
||||||
err: retryAttempt.error,
|
|
||||||
},
|
|
||||||
'Rotated Claude account also threw',
|
|
||||||
);
|
|
||||||
return 'error';
|
|
||||||
}
|
|
||||||
|
|
||||||
const retryOutput = retryAttempt.output;
|
|
||||||
if (!retryOutput) {
|
|
||||||
logger.error(
|
|
||||||
{
|
|
||||||
chatJid,
|
|
||||||
group: group.name,
|
|
||||||
groupFolder: group.folder,
|
|
||||||
runId,
|
|
||||||
provider: 'claude',
|
|
||||||
},
|
|
||||||
'Rotated Claude account produced no output object',
|
|
||||||
);
|
|
||||||
return 'error';
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
|
||||||
!retryAttempt.sawOutput &&
|
|
||||||
retryAttempt.streamedTriggerReason &&
|
|
||||||
retryOutput.status !== 'error'
|
|
||||||
) {
|
|
||||||
trigger = {
|
|
||||||
reason: retryAttempt.streamedTriggerReason.reason,
|
|
||||||
retryAfterMs: retryAttempt.streamedTriggerReason.retryAfterMs,
|
|
||||||
};
|
};
|
||||||
lastRotationMessage =
|
},
|
||||||
typeof retryOutput.result === 'string'
|
logCtx,
|
||||||
? retryOutput.result
|
rotationMessage,
|
||||||
: undefined;
|
);
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
switch (outcome.type) {
|
||||||
!retryAttempt.sawOutput &&
|
case 'success':
|
||||||
retryAttempt.sawSuccessNullResultWithoutOutput
|
return 'success';
|
||||||
) {
|
case 'error':
|
||||||
return canFallback
|
|
||||||
? runFallbackAttempt('success-null-result')
|
|
||||||
: 'error';
|
|
||||||
}
|
|
||||||
|
|
||||||
if (retryOutput.status === 'error') {
|
|
||||||
if (!retryAttempt.sawOutput) {
|
|
||||||
const retryTrigger = retryAttempt.streamedTriggerReason
|
|
||||||
? {
|
|
||||||
shouldFallback: true,
|
|
||||||
reason: retryAttempt.streamedTriggerReason.reason,
|
|
||||||
retryAfterMs: retryAttempt.streamedTriggerReason.retryAfterMs,
|
|
||||||
}
|
|
||||||
: detectFallbackTrigger(retryOutput.error);
|
|
||||||
if (retryTrigger.shouldFallback) {
|
|
||||||
trigger = {
|
|
||||||
reason: retryTrigger.reason,
|
|
||||||
retryAfterMs: retryTrigger.retryAfterMs,
|
|
||||||
};
|
|
||||||
lastRotationMessage = retryOutput.error ?? undefined;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.error(
|
|
||||||
{
|
|
||||||
group: group.name,
|
|
||||||
chatJid,
|
|
||||||
runId,
|
|
||||||
provider: 'claude',
|
|
||||||
error: retryOutput.error,
|
|
||||||
},
|
|
||||||
'Rotated Claude account failed',
|
|
||||||
);
|
|
||||||
return 'error';
|
return 'error';
|
||||||
}
|
case 'no-fallback':
|
||||||
|
return 'error';
|
||||||
markTokenHealthy();
|
case 'needs-fallback':
|
||||||
return 'success';
|
if (outcome.trigger.reason === 'success-null-result') {
|
||||||
|
return canFallback
|
||||||
|
? runFallbackAttempt('success-null-result')
|
||||||
|
: 'error';
|
||||||
|
}
|
||||||
|
if (!canFallback) {
|
||||||
|
logger.warn(
|
||||||
|
{ ...logCtx, reason: outcome.trigger.reason },
|
||||||
|
'All Claude tokens exhausted and fallback disabled',
|
||||||
|
);
|
||||||
|
return 'error';
|
||||||
|
}
|
||||||
|
return runFallbackAttempt(
|
||||||
|
outcome.trigger.reason,
|
||||||
|
outcome.trigger.retryAfterMs,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Usage exhausted or auth-expired: don't fall back to Kimi — log only
|
|
||||||
if (
|
|
||||||
trigger.reason === 'usage-exhausted' ||
|
|
||||||
trigger.reason === 'auth-expired'
|
|
||||||
) {
|
|
||||||
markPrimaryCooldown(trigger.reason, trigger.retryAfterMs);
|
|
||||||
logger.info(
|
|
||||||
{ chatJid, group: group.name, runId, reason: trigger.reason },
|
|
||||||
`All Claude tokens ${trigger.reason}, silently skipping (no Kimi fallback)`,
|
|
||||||
);
|
|
||||||
return 'error';
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!canFallback) {
|
|
||||||
logger.warn(
|
|
||||||
{ chatJid, group: group.name, runId, reason: trigger.reason },
|
|
||||||
'All Claude tokens exhausted and fallback disabled',
|
|
||||||
);
|
|
||||||
return 'error';
|
|
||||||
}
|
|
||||||
|
|
||||||
return runFallbackAttempt(trigger.reason, trigger.retryAfterMs);
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const provider = canFallback ? await getActiveProvider() : 'claude';
|
const provider = canFallback ? await getActiveProvider() : 'claude';
|
||||||
@@ -657,10 +549,7 @@ export async function runAgentForGroup(
|
|||||||
provider === 'claude' &&
|
provider === 'claude' &&
|
||||||
!primaryAttempt.sawOutput
|
!primaryAttempt.sawOutput
|
||||||
) {
|
) {
|
||||||
const errMsg =
|
const errMsg = getErrorMessage(primaryAttempt.error);
|
||||||
primaryAttempt.error instanceof Error
|
|
||||||
? primaryAttempt.error.message
|
|
||||||
: String(primaryAttempt.error);
|
|
||||||
const trigger = primaryAttempt.streamedTriggerReason
|
const trigger = primaryAttempt.streamedTriggerReason
|
||||||
? {
|
? {
|
||||||
shouldFallback: true,
|
shouldFallback: true,
|
||||||
@@ -680,10 +569,7 @@ export async function runAgentForGroup(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (!isClaudeCodeAgent) {
|
if (!isClaudeCodeAgent) {
|
||||||
const errMsg =
|
const errMsg = getErrorMessage(primaryAttempt.error);
|
||||||
primaryAttempt.error instanceof Error
|
|
||||||
? primaryAttempt.error.message
|
|
||||||
: String(primaryAttempt.error);
|
|
||||||
const trigger = detectCodexRotationTrigger(errMsg);
|
const trigger = detectCodexRotationTrigger(errMsg);
|
||||||
if (trigger.shouldRotate && getCodexAccountCount() > 1) {
|
if (trigger.shouldRotate && getCodexAccountCount() > 1) {
|
||||||
return retryCodexWithRotation({ reason: trigger.reason }, errMsg);
|
return retryCodexWithRotation({ reason: trigger.reason }, errMsg);
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { AgentOutput } from './agent-runner.js';
|
import { AgentOutput } from './agent-runner.js';
|
||||||
|
import { getErrorMessage } from './utils.js';
|
||||||
import {
|
import {
|
||||||
getMessagesSinceSeq,
|
getMessagesSinceSeq,
|
||||||
getNewMessagesBySeq,
|
getNewMessagesBySeq,
|
||||||
@@ -119,7 +120,7 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
);
|
);
|
||||||
return true;
|
return true;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
const errorMessage = getErrorMessage(err);
|
||||||
markWorkItemDeliveryRetry(item.id, errorMessage);
|
markWorkItemDeliveryRetry(item.id, errorMessage);
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -3,9 +3,14 @@ import { logger } from './logger.js';
|
|||||||
import { formatOutbound } from './router.js';
|
import { formatOutbound } from './router.js';
|
||||||
import { shouldResetSessionOnAgentFailure } from './session-recovery.js';
|
import { shouldResetSessionOnAgentFailure } from './session-recovery.js';
|
||||||
import { TASK_STATUS_MESSAGE_PREFIX } from './task-watch-status.js';
|
import { TASK_STATUS_MESSAGE_PREFIX } from './task-watch-status.js';
|
||||||
import { type Channel, type RegisteredGroup } from './types.js';
|
import { formatElapsedKorean } from './utils.js';
|
||||||
|
import {
|
||||||
|
type Channel,
|
||||||
|
type RegisteredGroup,
|
||||||
|
type VisiblePhase,
|
||||||
|
} from './types.js';
|
||||||
|
|
||||||
export type VisiblePhase = 'silent' | 'progress' | 'final';
|
export type { VisiblePhase };
|
||||||
|
|
||||||
interface SubagentTrack {
|
interface SubagentTrack {
|
||||||
label: string;
|
label: string;
|
||||||
@@ -324,20 +329,12 @@ export class MessageTurnController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private renderProgressMessage(text: string): string {
|
private renderProgressMessage(text: string): string {
|
||||||
const elapsedSeconds =
|
const elapsedMs =
|
||||||
this.progressStartedAt === null
|
this.progressStartedAt === null
|
||||||
? 0
|
? 0
|
||||||
: Math.floor((Date.now() - this.progressStartedAt) / 5_000) * 5;
|
: Math.floor((Date.now() - this.progressStartedAt) / 5_000) * 5000;
|
||||||
const hours = Math.floor(elapsedSeconds / 3600);
|
|
||||||
const minutes = Math.floor((elapsedSeconds % 3600) / 60);
|
|
||||||
const seconds = elapsedSeconds % 60;
|
|
||||||
const elapsedParts: string[] = [];
|
|
||||||
|
|
||||||
if (hours > 0) elapsedParts.push(`${hours}시간`);
|
const suffix = `\n\n${formatElapsedKorean(elapsedMs)}`;
|
||||||
if (minutes > 0) elapsedParts.push(`${minutes}분`);
|
|
||||||
elapsedParts.push(`${seconds}초`);
|
|
||||||
|
|
||||||
const suffix = `\n\n${elapsedParts.join(' ')}`;
|
|
||||||
let body: string;
|
let body: string;
|
||||||
|
|
||||||
if (this.subagents.size > 1) {
|
if (this.subagents.size > 1) {
|
||||||
|
|||||||
169
src/provider-retry.ts
Normal file
169
src/provider-retry.ts
Normal file
@@ -0,0 +1,169 @@
|
|||||||
|
/**
|
||||||
|
* Shared Claude retry-with-rotation loop (SSOT).
|
||||||
|
*
|
||||||
|
* Extracted from message-agent-executor.ts and task-scheduler.ts
|
||||||
|
* to eliminate the ~255-line structural duplication.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { shouldRotateClaudeToken } from './agent-error-detection.js';
|
||||||
|
import { logger } from './logger.js';
|
||||||
|
import { getErrorMessage } from './utils.js';
|
||||||
|
import { detectFallbackTrigger, markPrimaryCooldown } from './provider-fallback.js';
|
||||||
|
import { rotateToken, getTokenCount, markTokenHealthy } from './token-rotation.js';
|
||||||
|
|
||||||
|
// ── Types ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export interface TriggerInfo {
|
||||||
|
reason: string;
|
||||||
|
retryAfterMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RotationAttemptResult {
|
||||||
|
output?: { status: string; result?: string | null; error?: string | null };
|
||||||
|
thrownError?: unknown;
|
||||||
|
sawOutput: boolean;
|
||||||
|
sawSuccessNullResult?: boolean;
|
||||||
|
streamedTriggerReason?: TriggerInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type RotationOutcome =
|
||||||
|
| { type: 'success' }
|
||||||
|
| { type: 'error'; message?: string }
|
||||||
|
| { type: 'needs-fallback'; trigger: TriggerInfo }
|
||||||
|
| { type: 'no-fallback'; trigger: TriggerInfo }; // usage-exhausted/auth-expired
|
||||||
|
|
||||||
|
// ── Shared rotation loop ─────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Retry a Claude request by rotating through available tokens.
|
||||||
|
*
|
||||||
|
* Returns a discriminated outcome — the caller decides what to do
|
||||||
|
* with 'needs-fallback' (e.g. run Kimi fallback) or 'no-fallback'.
|
||||||
|
*/
|
||||||
|
export async function runClaudeRotationLoop(
|
||||||
|
initialTrigger: TriggerInfo,
|
||||||
|
runAttempt: () => Promise<RotationAttemptResult>,
|
||||||
|
logContext: Record<string, unknown>,
|
||||||
|
rotationMessage?: string,
|
||||||
|
): Promise<RotationOutcome> {
|
||||||
|
let trigger = initialTrigger;
|
||||||
|
let lastRotationMessage = rotationMessage;
|
||||||
|
|
||||||
|
while (
|
||||||
|
shouldRotateClaudeToken(trigger.reason) &&
|
||||||
|
getTokenCount() > 1 &&
|
||||||
|
rotateToken(lastRotationMessage, { ignoreRateLimits: true })
|
||||||
|
) {
|
||||||
|
logger.info(
|
||||||
|
{ ...logContext, reason: trigger.reason },
|
||||||
|
'Claude rate-limited, retrying with rotated account',
|
||||||
|
);
|
||||||
|
|
||||||
|
const attempt = await runAttempt();
|
||||||
|
|
||||||
|
// ── Thrown error (exception from spawn/process) ──
|
||||||
|
if (attempt.thrownError) {
|
||||||
|
if (!attempt.sawOutput) {
|
||||||
|
const errMsg = getErrorMessage(attempt.thrownError);
|
||||||
|
const retryTrigger = attempt.streamedTriggerReason
|
||||||
|
? {
|
||||||
|
shouldFallback: true,
|
||||||
|
reason: attempt.streamedTriggerReason.reason,
|
||||||
|
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
|
||||||
|
}
|
||||||
|
: detectFallbackTrigger(errMsg);
|
||||||
|
if (retryTrigger.shouldFallback) {
|
||||||
|
trigger = {
|
||||||
|
reason: retryTrigger.reason,
|
||||||
|
retryAfterMs: retryTrigger.retryAfterMs,
|
||||||
|
};
|
||||||
|
lastRotationMessage = errMsg;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.error(
|
||||||
|
{ ...logContext, provider: 'claude', err: attempt.thrownError },
|
||||||
|
'Rotated Claude account also threw',
|
||||||
|
);
|
||||||
|
return { type: 'error' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const output = attempt.output;
|
||||||
|
if (!output) {
|
||||||
|
logger.error(
|
||||||
|
{ ...logContext, provider: 'claude' },
|
||||||
|
'Rotated Claude account produced no output object',
|
||||||
|
);
|
||||||
|
return { type: 'error' };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Streamed trigger in non-error success ──
|
||||||
|
if (
|
||||||
|
!attempt.sawOutput &&
|
||||||
|
attempt.streamedTriggerReason &&
|
||||||
|
output.status !== 'error'
|
||||||
|
) {
|
||||||
|
trigger = {
|
||||||
|
reason: attempt.streamedTriggerReason.reason,
|
||||||
|
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
|
||||||
|
};
|
||||||
|
lastRotationMessage =
|
||||||
|
typeof output.result === 'string' ? output.result : undefined;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Success with null result (MAE-specific, TaskScheduler ignores) ──
|
||||||
|
if (!attempt.sawOutput && attempt.sawSuccessNullResult) {
|
||||||
|
return { type: 'needs-fallback', trigger: { reason: 'success-null-result' } };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Error status ──
|
||||||
|
if (output.status === 'error') {
|
||||||
|
if (!attempt.sawOutput) {
|
||||||
|
const retryTrigger = attempt.streamedTriggerReason
|
||||||
|
? {
|
||||||
|
shouldFallback: true,
|
||||||
|
reason: attempt.streamedTriggerReason.reason,
|
||||||
|
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
|
||||||
|
}
|
||||||
|
: detectFallbackTrigger(output.error);
|
||||||
|
if (retryTrigger.shouldFallback) {
|
||||||
|
trigger = {
|
||||||
|
reason: retryTrigger.reason,
|
||||||
|
retryAfterMs: retryTrigger.retryAfterMs,
|
||||||
|
};
|
||||||
|
lastRotationMessage = output.error ?? undefined;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.error(
|
||||||
|
{ ...logContext, provider: 'claude', error: output.error },
|
||||||
|
'Rotated Claude account failed',
|
||||||
|
);
|
||||||
|
return { type: 'error' };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Success ──
|
||||||
|
markTokenHealthy();
|
||||||
|
return { type: 'success' };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── All tokens exhausted ──
|
||||||
|
|
||||||
|
// Usage exhausted or auth-expired: don't fall back to Kimi
|
||||||
|
if (
|
||||||
|
trigger.reason === 'usage-exhausted' ||
|
||||||
|
trigger.reason === 'auth-expired'
|
||||||
|
) {
|
||||||
|
markPrimaryCooldown(trigger.reason, trigger.retryAfterMs);
|
||||||
|
logger.info(
|
||||||
|
{ ...logContext, reason: trigger.reason },
|
||||||
|
`All Claude tokens ${trigger.reason}, silently skipping (no Kimi fallback)`,
|
||||||
|
);
|
||||||
|
return { type: 'no-fallback', trigger };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { type: 'needs-fallback', trigger };
|
||||||
|
}
|
||||||
@@ -5,6 +5,7 @@ import path from 'path';
|
|||||||
import { DATA_DIR, SERVICE_ID, TIMEZONE } from './config.js';
|
import { DATA_DIR, SERVICE_ID, TIMEZONE } from './config.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import type { RegisteredGroup } from './types.js';
|
import type { RegisteredGroup } from './types.js';
|
||||||
|
import { readJsonFile, writeJsonFile } from './utils.js';
|
||||||
|
|
||||||
export interface RestartInterruptedGroup {
|
export interface RestartInterruptedGroup {
|
||||||
chatJid: string;
|
chatJid: string;
|
||||||
@@ -61,16 +62,14 @@ function getMainGroupJid(
|
|||||||
}
|
}
|
||||||
|
|
||||||
function readRestartContextFile(filePath: string): RestartContext | null {
|
function readRestartContextFile(filePath: string): RestartContext | null {
|
||||||
if (!fs.existsSync(filePath)) return null;
|
const data = readJsonFile<RestartContext>(filePath);
|
||||||
try {
|
if (data === null && fs.existsSync(filePath)) {
|
||||||
return JSON.parse(fs.readFileSync(filePath, 'utf8')) as RestartContext;
|
|
||||||
} catch (err) {
|
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{ err, filePath },
|
{ filePath },
|
||||||
'Failed to parse restart context file; ignoring invalid content',
|
'Failed to parse restart context file; ignoring invalid content',
|
||||||
);
|
);
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
return data;
|
||||||
}
|
}
|
||||||
|
|
||||||
function getLatestDistBuildTime(): number | null {
|
function getLatestDistBuildTime(): number | null {
|
||||||
@@ -134,7 +133,7 @@ export function writeRestartContext(
|
|||||||
const writtenPaths: string[] = [];
|
const writtenPaths: string[] = [];
|
||||||
for (const serviceId of serviceIds) {
|
for (const serviceId of serviceIds) {
|
||||||
const filePath = getRestartContextPath(serviceId);
|
const filePath = getRestartContextPath(serviceId);
|
||||||
fs.writeFileSync(filePath, JSON.stringify(payload, null, 2), 'utf8');
|
writeJsonFile(filePath, payload, true);
|
||||||
writtenPaths.push(filePath);
|
writtenPaths.push(filePath);
|
||||||
}
|
}
|
||||||
return writtenPaths;
|
return writtenPaths;
|
||||||
@@ -175,7 +174,7 @@ export function writeShutdownRestartContext(
|
|||||||
interruptedGroups: mergedInterrupted,
|
interruptedGroups: mergedInterrupted,
|
||||||
};
|
};
|
||||||
|
|
||||||
fs.writeFileSync(filePath, JSON.stringify(payload, null, 2), 'utf8');
|
writeJsonFile(filePath, payload, true);
|
||||||
writtenPaths.push(filePath);
|
writtenPaths.push(filePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -205,24 +204,19 @@ export function buildInterruptedRestartAnnouncement(
|
|||||||
export function consumeRestartContext(): RestartContext | null {
|
export function consumeRestartContext(): RestartContext | null {
|
||||||
const filePath = getRestartContextPath();
|
const filePath = getRestartContextPath();
|
||||||
if (!fs.existsSync(filePath)) return null;
|
if (!fs.existsSync(filePath)) return null;
|
||||||
try {
|
const parsed = readJsonFile<RestartContext>(filePath);
|
||||||
const parsed = JSON.parse(
|
if (!parsed) {
|
||||||
fs.readFileSync(filePath, 'utf8'),
|
|
||||||
) as RestartContext;
|
|
||||||
fs.unlinkSync(filePath);
|
|
||||||
return parsed;
|
|
||||||
} catch (err) {
|
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{ err, filePath },
|
{ filePath },
|
||||||
'Failed to read restart context; removing invalid file',
|
'Failed to read restart context; removing invalid file',
|
||||||
);
|
);
|
||||||
try {
|
|
||||||
fs.unlinkSync(filePath);
|
|
||||||
} catch {
|
|
||||||
// Ignore cleanup failure.
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
try {
|
||||||
|
fs.unlinkSync(filePath);
|
||||||
|
} catch {
|
||||||
|
// Ignore cleanup failure.
|
||||||
|
}
|
||||||
|
return parsed;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildRestartAnnouncement(context: RestartContext): string {
|
export function buildRestartAnnouncement(context: RestartContext): string {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import fs from 'fs';
|
|||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
|
||||||
import { CACHE_DIR } from './config.js';
|
import { CACHE_DIR } from './config.js';
|
||||||
|
import { writeJsonFile } from './utils.js';
|
||||||
import type { GroupStatus } from './group-queue.js';
|
import type { GroupStatus } from './group-queue.js';
|
||||||
import type { AgentType } from './types.js';
|
import type { AgentType } from './types.js';
|
||||||
|
|
||||||
@@ -32,7 +33,7 @@ export function writeStatusSnapshot(snapshot: StatusSnapshot): void {
|
|||||||
`${snapshot.agentType}.json`,
|
`${snapshot.agentType}.json`,
|
||||||
);
|
);
|
||||||
const tempPath = `${targetPath}.tmp`;
|
const tempPath = `${targetPath}.tmp`;
|
||||||
fs.writeFileSync(tempPath, JSON.stringify(snapshot, null, 2));
|
writeJsonFile(tempPath, snapshot, true);
|
||||||
fs.renameSync(tempPath, targetPath);
|
fs.renameSync(tempPath, targetPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { ChildProcess } from 'child_process';
|
|||||||
import { CronExpressionParser } from 'cron-parser';
|
import { CronExpressionParser } from 'cron-parser';
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
import { getErrorMessage } from './utils.js';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
ASSISTANT_NAME,
|
ASSISTANT_NAME,
|
||||||
@@ -34,7 +35,6 @@ import { createTaskStatusTracker } from './task-status-tracker.js';
|
|||||||
import {
|
import {
|
||||||
isClaudeAuthExpiredMessage,
|
isClaudeAuthExpiredMessage,
|
||||||
isClaudeUsageExhaustedMessage,
|
isClaudeUsageExhaustedMessage,
|
||||||
shouldRotateClaudeToken,
|
|
||||||
} from './agent-error-detection.js';
|
} from './agent-error-detection.js';
|
||||||
import {
|
import {
|
||||||
detectFallbackTrigger,
|
detectFallbackTrigger,
|
||||||
@@ -46,6 +46,7 @@ import {
|
|||||||
isUsageExhausted,
|
isUsageExhausted,
|
||||||
markPrimaryCooldown,
|
markPrimaryCooldown,
|
||||||
} from './provider-fallback.js';
|
} from './provider-fallback.js';
|
||||||
|
import { runClaudeRotationLoop } from './provider-retry.js';
|
||||||
import {
|
import {
|
||||||
detectCodexRotationTrigger,
|
detectCodexRotationTrigger,
|
||||||
rotateCodexToken,
|
rotateCodexToken,
|
||||||
@@ -53,9 +54,9 @@ import {
|
|||||||
markCodexTokenHealthy,
|
markCodexTokenHealthy,
|
||||||
} from './codex-token-rotation.js';
|
} from './codex-token-rotation.js';
|
||||||
import {
|
import {
|
||||||
rotateToken,
|
|
||||||
getTokenCount,
|
getTokenCount,
|
||||||
markTokenHealthy,
|
markTokenHealthy,
|
||||||
|
rotateToken,
|
||||||
} from './token-rotation.js';
|
} from './token-rotation.js';
|
||||||
import {
|
import {
|
||||||
evaluateTaskSuspension,
|
evaluateTaskSuspension,
|
||||||
@@ -222,7 +223,7 @@ async function runTask(
|
|||||||
try {
|
try {
|
||||||
context = resolveTaskExecutionContext(task, deps);
|
context = resolveTaskExecutionContext(task, deps);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const error = err instanceof Error ? err.message : String(err);
|
const error = getErrorMessage(err);
|
||||||
if (error.startsWith('Group not found:')) {
|
if (error.startsWith('Group not found:')) {
|
||||||
logger.error(
|
logger.error(
|
||||||
{ taskId: task.id, groupFolder: task.group_folder, error },
|
{ taskId: task.id, groupFolder: task.group_folder, error },
|
||||||
@@ -441,83 +442,44 @@ async function runTask(
|
|||||||
},
|
},
|
||||||
rotationMessage?: string,
|
rotationMessage?: string,
|
||||||
): Promise<void> => {
|
): Promise<void> => {
|
||||||
let trigger = initialTrigger;
|
const logCtx = {
|
||||||
let lastRotationMessage = rotationMessage;
|
taskId: task.id,
|
||||||
|
group: context.group.name,
|
||||||
|
groupFolder: task.group_folder,
|
||||||
|
};
|
||||||
|
|
||||||
while (
|
const outcome = await runClaudeRotationLoop(
|
||||||
shouldRotateClaudeToken(trigger.reason) &&
|
initialTrigger,
|
||||||
getTokenCount() > 1 &&
|
async () => {
|
||||||
rotateToken(lastRotationMessage, { ignoreRateLimits: true })
|
const attempt = await runTaskAttempt('claude');
|
||||||
) {
|
result = attempt.attemptResult;
|
||||||
logger.info(
|
error = attempt.attemptError;
|
||||||
{
|
return {
|
||||||
taskId: task.id,
|
output: attempt.output,
|
||||||
group: context.group.name,
|
sawOutput: attempt.sawOutput,
|
||||||
groupFolder: task.group_folder,
|
streamedTriggerReason: attempt.streamedTriggerReason,
|
||||||
reason: trigger.reason,
|
|
||||||
},
|
|
||||||
'Scheduled task Claude rate-limited, retrying with rotated account',
|
|
||||||
);
|
|
||||||
|
|
||||||
const retryAttempt = await runTaskAttempt('claude');
|
|
||||||
result = retryAttempt.attemptResult;
|
|
||||||
error = retryAttempt.attemptError;
|
|
||||||
|
|
||||||
if (
|
|
||||||
retryAttempt.streamedTriggerReason &&
|
|
||||||
!retryAttempt.sawOutput &&
|
|
||||||
retryAttempt.output.status !== 'error'
|
|
||||||
) {
|
|
||||||
trigger = {
|
|
||||||
reason: retryAttempt.streamedTriggerReason.reason,
|
|
||||||
retryAfterMs: retryAttempt.streamedTriggerReason.retryAfterMs,
|
|
||||||
};
|
};
|
||||||
lastRotationMessage =
|
},
|
||||||
typeof retryAttempt.output.result === 'string'
|
logCtx,
|
||||||
? retryAttempt.output.result
|
rotationMessage,
|
||||||
: undefined;
|
);
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (retryAttempt.output.status === 'error' && !retryAttempt.sawOutput) {
|
switch (outcome.type) {
|
||||||
const retryTrigger = retryAttempt.streamedTriggerReason
|
case 'success':
|
||||||
? {
|
|
||||||
shouldFallback: true,
|
|
||||||
reason: retryAttempt.streamedTriggerReason.reason,
|
|
||||||
retryAfterMs: retryAttempt.streamedTriggerReason.retryAfterMs,
|
|
||||||
}
|
|
||||||
: detectFallbackTrigger(retryAttempt.attemptError);
|
|
||||||
if (retryTrigger.shouldFallback) {
|
|
||||||
trigger = {
|
|
||||||
reason: retryTrigger.reason,
|
|
||||||
retryAfterMs: retryTrigger.retryAfterMs,
|
|
||||||
};
|
|
||||||
lastRotationMessage = retryAttempt.attemptError || undefined;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (retryAttempt.output.status === 'success') {
|
|
||||||
markTokenHealthy();
|
|
||||||
error = null;
|
error = null;
|
||||||
return;
|
return;
|
||||||
}
|
case 'error':
|
||||||
|
return;
|
||||||
return;
|
case 'no-fallback':
|
||||||
|
error = `Claude ${outcome.trigger.reason}`;
|
||||||
|
return;
|
||||||
|
case 'needs-fallback':
|
||||||
|
await runFallbackTaskAttempt(
|
||||||
|
outcome.trigger.reason,
|
||||||
|
outcome.trigger.retryAfterMs,
|
||||||
|
);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Usage exhausted: don't fall back to Kimi — just mark cooldown and skip
|
|
||||||
if (trigger.reason === 'usage-exhausted') {
|
|
||||||
markPrimaryCooldown(trigger.reason, trigger.retryAfterMs);
|
|
||||||
logger.info(
|
|
||||||
{ taskId: task.id, group: context.group.name },
|
|
||||||
'All Claude tokens usage-exhausted, skipping Kimi fallback for scheduled task',
|
|
||||||
);
|
|
||||||
error = 'Claude usage exhausted';
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
await runFallbackTaskAttempt(trigger.reason, trigger.retryAfterMs);
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const provider = canFallback ? await getActiveProvider() : 'claude';
|
const provider = canFallback ? await getActiveProvider() : 'claude';
|
||||||
@@ -569,7 +531,7 @@ async function runTask(
|
|||||||
'Task completed',
|
'Task completed',
|
||||||
);
|
);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
error = err instanceof Error ? err.message : String(err);
|
error = getErrorMessage(err);
|
||||||
logger.error({ taskId: task.id, error }, 'Task failed');
|
logger.error({ taskId: task.id, error }, 'Task failed');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { TIMEZONE } from './config.js';
|
import { TIMEZONE } from './config.js';
|
||||||
import type { ScheduledTask } from './types.js';
|
import type { ScheduledTask } from './types.js';
|
||||||
|
import { formatElapsedKorean } from './utils.js';
|
||||||
|
|
||||||
export type WatcherStatusPhase =
|
export type WatcherStatusPhase =
|
||||||
| 'checking'
|
| 'checking'
|
||||||
@@ -65,16 +66,7 @@ function formatElapsedLabel(
|
|||||||
0,
|
0,
|
||||||
new Date(checkedAtIso).getTime() - new Date(startedAtIso).getTime(),
|
new Date(checkedAtIso).getTime() - new Date(startedAtIso).getTime(),
|
||||||
);
|
);
|
||||||
const totalSeconds = Math.floor(elapsedMs / 1000);
|
return formatElapsedKorean(elapsedMs);
|
||||||
const hours = Math.floor(totalSeconds / 3600);
|
|
||||||
const minutes = Math.floor((totalSeconds % 3600) / 60);
|
|
||||||
const seconds = totalSeconds % 60;
|
|
||||||
|
|
||||||
const parts: string[] = [];
|
|
||||||
if (hours > 0) parts.push(`${hours}시간`);
|
|
||||||
if (minutes > 0) parts.push(`${minutes}분`);
|
|
||||||
parts.push(`${seconds}초`);
|
|
||||||
return parts.join(' ');
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function renderWatchCiStatusMessage(args: {
|
export function renderWatchCiStatusMessage(args: {
|
||||||
|
|||||||
@@ -13,6 +13,7 @@
|
|||||||
*/
|
*/
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import os from 'os';
|
import os from 'os';
|
||||||
|
import { getErrorMessage, readJsonFile } from './utils.js';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
@@ -72,13 +73,12 @@ function getCredentialsPath(accountIndex: number): string {
|
|||||||
|
|
||||||
function readCredentials(accountIndex: number): CredentialsFile | null {
|
function readCredentials(accountIndex: number): CredentialsFile | null {
|
||||||
const credsPath = getCredentialsPath(accountIndex);
|
const credsPath = getCredentialsPath(accountIndex);
|
||||||
try {
|
if (!fs.existsSync(credsPath)) return null;
|
||||||
if (!fs.existsSync(credsPath)) return null;
|
const data = readJsonFile<CredentialsFile>(credsPath);
|
||||||
return JSON.parse(fs.readFileSync(credsPath, 'utf-8'));
|
if (!data) {
|
||||||
} catch (err) {
|
logger.warn({ accountIndex }, 'Failed to read Claude credentials');
|
||||||
logger.warn({ err, accountIndex }, 'Failed to read Claude credentials');
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
return data;
|
||||||
}
|
}
|
||||||
|
|
||||||
function writeCredentials(accountIndex: number, creds: CredentialsFile): void {
|
function writeCredentials(accountIndex: number, creds: CredentialsFile): void {
|
||||||
@@ -121,7 +121,7 @@ function syncToSessionDirs(credsPath: string): void {
|
|||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{ err: err instanceof Error ? err.message : String(err) },
|
{ err: getErrorMessage(err) },
|
||||||
'Failed to sync credentials to sessions',
|
'Failed to sync credentials to sessions',
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -165,7 +165,7 @@ function updateEnvTokens(): void {
|
|||||||
logger.debug('Updated .env with refreshed tokens');
|
logger.debug('Updated .env with refreshed tokens');
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{ err: err instanceof Error ? err.message : String(err) },
|
{ err: getErrorMessage(err) },
|
||||||
'Failed to update .env with refreshed tokens',
|
'Failed to update .env with refreshed tokens',
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -208,7 +208,7 @@ async function refreshToken(
|
|||||||
);
|
);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{ url: TOKEN_URL, err: err instanceof Error ? err.message : String(err) },
|
{ url: TOKEN_URL, err: getErrorMessage(err) },
|
||||||
'Token refresh request error',
|
'Token refresh request error',
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -276,7 +276,7 @@ async function checkAndRefreshAccount(
|
|||||||
logger.error(
|
logger.error(
|
||||||
{
|
{
|
||||||
accountIndex,
|
accountIndex,
|
||||||
err: err instanceof Error ? err.message : String(err),
|
err: getErrorMessage(err),
|
||||||
},
|
},
|
||||||
'Failed to refresh Claude OAuth token — manual re-login may be required',
|
'Failed to refresh Claude OAuth token — manual re-login may be required',
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ import {
|
|||||||
computeCooldownUntil,
|
computeCooldownUntil,
|
||||||
findNextAvailable,
|
findNextAvailable,
|
||||||
} from './token-rotation-base.js';
|
} from './token-rotation-base.js';
|
||||||
|
import { readJsonFile, writeJsonFile } from './utils.js';
|
||||||
|
|
||||||
const STATE_FILE = path.join(DATA_DIR, 'token-rotation-state.json');
|
const STATE_FILE = path.join(DATA_DIR, 'token-rotation-state.json');
|
||||||
|
|
||||||
@@ -67,42 +68,39 @@ function saveState(): void {
|
|||||||
currentIndex,
|
currentIndex,
|
||||||
rateLimits: tokens.map((t) => t.rateLimitedUntil),
|
rateLimits: tokens.map((t) => t.rateLimitedUntil),
|
||||||
};
|
};
|
||||||
fs.writeFileSync(STATE_FILE, JSON.stringify(state));
|
writeJsonFile(STATE_FILE, state);
|
||||||
} catch {
|
} catch {
|
||||||
/* best effort */
|
/* best effort */
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function loadState(): void {
|
function loadState(): void {
|
||||||
try {
|
const state = readJsonFile<{ currentIndex?: number; rateLimits?: (number | null)[] }>(STATE_FILE);
|
||||||
if (!fs.existsSync(STATE_FILE)) return;
|
if (!state) return;
|
||||||
const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
|
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
if (
|
if (
|
||||||
typeof state.currentIndex === 'number' &&
|
typeof state.currentIndex === 'number' &&
|
||||||
state.currentIndex < tokens.length
|
state.currentIndex < tokens.length
|
||||||
|
) {
|
||||||
|
currentIndex = state.currentIndex;
|
||||||
|
}
|
||||||
|
if (Array.isArray(state.rateLimits)) {
|
||||||
|
for (
|
||||||
|
let i = 0;
|
||||||
|
i < Math.min(state.rateLimits.length, tokens.length);
|
||||||
|
i++
|
||||||
) {
|
) {
|
||||||
currentIndex = state.currentIndex;
|
const until = state.rateLimits[i];
|
||||||
}
|
if (typeof until === 'number' && until > now) {
|
||||||
if (Array.isArray(state.rateLimits)) {
|
tokens[i].rateLimitedUntil = until;
|
||||||
for (
|
|
||||||
let i = 0;
|
|
||||||
i < Math.min(state.rateLimits.length, tokens.length);
|
|
||||||
i++
|
|
||||||
) {
|
|
||||||
const until = state.rateLimits[i];
|
|
||||||
if (typeof until === 'number' && until > now) {
|
|
||||||
tokens[i].rateLimitedUntil = until;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
logger.info(
|
|
||||||
{ currentIndex, tokenCount: tokens.length },
|
|
||||||
'Token rotation state restored',
|
|
||||||
);
|
|
||||||
} catch {
|
|
||||||
/* start fresh */
|
|
||||||
}
|
}
|
||||||
|
logger.info(
|
||||||
|
{ currentIndex, tokenCount: tokens.length },
|
||||||
|
'Token rotation state restored',
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Get the current active token. */
|
/** Get the current active token. */
|
||||||
|
|||||||
12
src/types.ts
12
src/types.ts
@@ -5,10 +5,22 @@ export interface AgentConfig {
|
|||||||
codexEffort?: string;
|
codexEffort?: string;
|
||||||
claudeModel?: string;
|
claudeModel?: string;
|
||||||
claudeEffort?: string;
|
claudeEffort?: string;
|
||||||
|
claudeThinking?: 'adaptive' | 'enabled' | 'disabled';
|
||||||
|
claudeThinkingBudget?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type AgentType = 'claude-code' | 'codex';
|
export type AgentType = 'claude-code' | 'codex';
|
||||||
|
|
||||||
|
/** Phase of agent output as emitted by the runner. */
|
||||||
|
export type AgentOutputPhase =
|
||||||
|
| 'progress'
|
||||||
|
| 'final'
|
||||||
|
| 'tool-activity'
|
||||||
|
| 'intermediate';
|
||||||
|
|
||||||
|
/** Phase as visible in the UI (mapped from AgentOutputPhase). */
|
||||||
|
export type VisiblePhase = 'silent' | 'progress' | 'final';
|
||||||
|
|
||||||
export interface RegisteredGroup {
|
export interface RegisteredGroup {
|
||||||
name: string;
|
name: string;
|
||||||
folder: string;
|
folder: string;
|
||||||
|
|||||||
69
src/utils.ts
Normal file
69
src/utils.ts
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
/**
|
||||||
|
* Shared utilities (SSOT).
|
||||||
|
*
|
||||||
|
* Small helpers that were previously copy-pasted across 10+ files.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import fs from 'fs';
|
||||||
|
|
||||||
|
// ── Error handling ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Extract a human-readable message from an unknown caught value. */
|
||||||
|
export function getErrorMessage(err: unknown): string {
|
||||||
|
return err instanceof Error ? err.message : String(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── JSON file I/O ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Read and parse a JSON file. Returns null on any failure. */
|
||||||
|
export function readJsonFile<T = unknown>(filePath: string): T | null {
|
||||||
|
try {
|
||||||
|
return JSON.parse(fs.readFileSync(filePath, 'utf-8')) as T;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Write data as JSON to a file. */
|
||||||
|
export function writeJsonFile(
|
||||||
|
filePath: string,
|
||||||
|
data: unknown,
|
||||||
|
pretty = false,
|
||||||
|
): void {
|
||||||
|
fs.writeFileSync(
|
||||||
|
filePath,
|
||||||
|
JSON.stringify(data, null, pretty ? 2 : undefined),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Fetch with timeout ──────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Wrapper around fetch() that aborts after timeoutMs. */
|
||||||
|
export async function fetchWithTimeout(
|
||||||
|
url: string,
|
||||||
|
init: RequestInit,
|
||||||
|
timeoutMs: number,
|
||||||
|
): Promise<Response> {
|
||||||
|
const controller = new AbortController();
|
||||||
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||||
|
try {
|
||||||
|
return await fetch(url, { ...init, signal: controller.signal });
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Time formatting ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Format milliseconds as Korean elapsed time (e.g. "1시간 2분 30초"). */
|
||||||
|
export function formatElapsedKorean(ms: number): string {
|
||||||
|
const totalSeconds = Math.floor(ms / 1000);
|
||||||
|
const hours = Math.floor(totalSeconds / 3600);
|
||||||
|
const minutes = Math.floor((totalSeconds % 3600) / 60);
|
||||||
|
const seconds = totalSeconds % 60;
|
||||||
|
const parts: string[] = [];
|
||||||
|
if (hours > 0) parts.push(`${hours}시간`);
|
||||||
|
if (minutes > 0) parts.push(`${minutes}분`);
|
||||||
|
parts.push(`${seconds}초`);
|
||||||
|
return parts.join(' ');
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user