chore: add state transition logging

This commit is contained in:
Eyejoker
2026-03-25 12:16:09 +09:00
parent cc7cfbbc5f
commit 18788f7f91
7 changed files with 207 additions and 25 deletions

View File

@@ -261,8 +261,10 @@ export function rotateCodexToken(
): boolean { ): boolean {
if (accounts.length <= 1) return false; if (accounts.length <= 1) return false;
const previousIndex = currentIndex;
const acct = accounts[currentIndex]; const acct = accounts[currentIndex];
acct.rateLimitedUntil = computeCooldownUntil(errorMessage); const cooldownUntil = computeCooldownUntil(errorMessage);
acct.rateLimitedUntil = cooldownUntil;
acct.lastUsagePct = 100; acct.lastUsagePct = 100;
// Extract reset time string from error for display // Extract reset time string from error for display
const retryAt = parseRetryAfterFromError(errorMessage); const retryAt = parseRetryAfterFromError(errorMessage);
@@ -276,9 +278,15 @@ export function rotateCodexToken(
currentIndex = nextIdx; currentIndex = nextIdx;
logger.info( logger.info(
{ {
accountIndex: currentIndex, transition: 'rotation:execute',
fromIndex: previousIndex,
toIndex: currentIndex,
totalAccounts: accounts.length, totalAccounts: accounts.length,
accountId: accounts[nextIdx].accountId, accountId: accounts[nextIdx].accountId,
ignoreRL: opts?.ignoreRateLimits ?? false,
cooldownUntil:
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
reason: errorMessage ?? null,
}, },
`Codex rotated to account #${currentIndex + 1}/${accounts.length}`, `Codex rotated to account #${currentIndex + 1}/${accounts.length}`,
); );
@@ -286,7 +294,18 @@ export function rotateCodexToken(
return true; return true;
} }
logger.warn('All Codex accounts are rate-limited'); logger.warn(
{
transition: 'rotation:skip',
fromIndex: previousIndex,
totalAccounts: accounts.length,
ignoreRL: opts?.ignoreRateLimits ?? false,
cooldownUntil:
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
reason: errorMessage ?? null,
},
'All Codex accounts are rate-limited',
);
return false; return false;
} }
@@ -353,7 +372,13 @@ export function updateCodexAccountUsage(
const nextIdx = findNextCodexAvailable(idx); const nextIdx = findNextCodexAvailable(idx);
if (nextIdx !== null && nextIdx !== idx) { if (nextIdx !== null && nextIdx !== idx) {
logger.info( logger.info(
{ from: idx + 1, to: nextIdx + 1, d7Pct }, {
transition: 'rotation:auto',
fromIndex: idx,
toIndex: nextIdx,
d7Pct,
accountId: acct.accountId,
},
`Codex auto-rotating: account #${idx + 1} at ${d7Pct}% 7d → #${nextIdx + 1}`, `Codex auto-rotating: account #${idx + 1} at ${d7Pct}% 7d → #${nextIdx + 1}`,
); );
currentIndex = nextIdx; currentIndex = nextIdx;
@@ -367,7 +392,17 @@ export function markCodexTokenHealthy(): void {
if (accounts.length === 0) return; if (accounts.length === 0) return;
const acct = accounts[currentIndex]; const acct = accounts[currentIndex];
if (acct?.rateLimitedUntil) { if (acct?.rateLimitedUntil) {
const previousCooldownUntil = acct.rateLimitedUntil;
acct.rateLimitedUntil = null; acct.rateLimitedUntil = null;
logger.info(
{
transition: 'rotation:clear-rate-limit',
accountIndex: currentIndex,
accountId: acct.accountId,
cooldownUntil: new Date(previousCooldownUntil).toISOString(),
},
'Cleared Codex account rate-limit state after successful response',
);
saveCodexState(); saveCodexState();
} }
} }

View File

@@ -507,17 +507,38 @@ export class GroupQueue {
this.activeCount++; this.activeCount++;
this.activeTaskCount++; this.activeTaskCount++;
logger.debug( logger.info(
{ groupJid, taskId: task.id, activeCount: this.activeCount }, {
transition: 'queue:task:start',
groupJid,
taskId: task.id,
activeCount: this.activeCount,
activeTaskCount: this.activeTaskCount,
},
'Running queued task', 'Running queued task',
); );
let outcome: 'success' | 'error' = 'success';
try { try {
await task.fn(); await task.fn();
} catch (err) { } catch (err) {
outcome = 'error';
logger.error({ groupJid, taskId: task.id, err }, 'Error running task'); logger.error({ groupJid, taskId: task.id, err }, 'Error running task');
} finally { } finally {
this.clearPostCloseTimers(state); this.clearPostCloseTimers(state);
const durationMs = state.startedAt ? Date.now() - state.startedAt : null;
logger.info(
{
transition: 'queue:task:finish',
groupJid,
taskId: task.id,
outcome,
durationMs,
pendingMessages: state.pendingMessages,
pendingTasks: state.pendingTasks.length,
},
'Finished queued task',
);
state.active = false; state.active = false;
state.isTaskProcess = false; state.isTaskProcess = false;
state.runningTaskId = null; state.runningTaskId = null;

View File

@@ -90,18 +90,31 @@ export function startIpcWatcher(deps: IpcDeps): void {
if (msg.type === 'message' && msg.chatJid && msg.text) { if (msg.type === 'message' && msg.chatJid && msg.text) {
// Authorization: verify this group can send to this chatJid // Authorization: verify this group can send to this chatJid
const targetGroup = registeredGroups[msg.chatJid]; const targetGroup = registeredGroups[msg.chatJid];
const isMainOverride = isMain === true;
if ( if (
isMain || isMainOverride ||
(targetGroup && targetGroup.folder === sourceGroup) (targetGroup && targetGroup.folder === sourceGroup)
) { ) {
await deps.sendMessage(msg.chatJid, msg.text); await deps.sendMessage(msg.chatJid, msg.text);
logger.info( logger.info(
{ chatJid: msg.chatJid, sourceGroup }, {
transition: 'ipc:auth:allow',
chatJid: msg.chatJid,
sourceGroup,
targetGroup: targetGroup?.folder ?? null,
isMainOverride,
},
'IPC message sent', 'IPC message sent',
); );
} else { } else {
logger.warn( logger.warn(
{ chatJid: msg.chatJid, sourceGroup }, {
transition: 'ipc:auth:deny',
chatJid: msg.chatJid,
sourceGroup,
targetGroup: targetGroup?.folder ?? null,
isMainOverride,
},
'Unauthorized IPC message attempt blocked', 'Unauthorized IPC message attempt blocked',
); );
} }

View File

@@ -376,7 +376,18 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
} finally { } finally {
// Safety net: always clear typing even if runAgent() or finish() throws. // Safety net: always clear typing even if runAgent() or finish() throws.
// Prevents stuck typing indicators when exceptions bypass the normal // Prevents stuck typing indicators when exceptions bypass the normal
// turnController.finish() setTyping(false) path. // turnController.finish() -> setTyping(false) path.
logger.debug(
{
transition: 'typing:off',
source: 'message-runtime:safety-net',
chatJid,
group: group.name,
groupFolder: group.folder,
runId,
},
'Typing indicator transition',
);
await channel.setTyping?.(chatJid, false); await channel.setTyping?.(chatJid, false);
} }
} }
@@ -530,6 +541,17 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
endSeq, endSeq,
); );
} }
logger.debug(
{
transition: 'typing:on',
source: 'follow-up-queued',
chatJid,
group: group.name,
groupFolder: group.folder,
endSeq: endSeq ?? null,
},
'Typing indicator transition',
);
await channel await channel
.setTyping?.(chatJid, true) .setTyping?.(chatJid, true)
?.catch((err) => ?.catch((err) =>

View File

@@ -56,9 +56,29 @@ export class MessageTurnController {
constructor(private readonly options: MessageTurnControllerOptions) {} constructor(private readonly options: MessageTurnControllerOptions) {}
private async setTyping(
isTyping: boolean,
source: string,
extra?: Record<string, unknown>,
): Promise<void> {
logger.debug(
{
transition: isTyping ? 'typing:on' : 'typing:off',
source,
chatJid: this.options.chatJid,
group: this.options.group.name,
groupFolder: this.options.group.folder,
runId: this.options.runId,
...extra,
},
'Typing indicator transition',
);
await this.options.channel.setTyping?.(this.options.chatJid, isTyping);
}
async start(): Promise<void> { async start(): Promise<void> {
this.resetIdleTimer(); this.resetIdleTimer();
await this.options.channel.setTyping?.(this.options.chatJid, true); await this.setTyping(true, 'turn:start');
} }
async handleOutput(result: AgentOutput): Promise<void> { async handleOutput(result: AgentOutput): Promise<void> {
@@ -274,7 +294,10 @@ export class MessageTurnController {
await this.finalizeProgressMessage(); await this.finalizeProgressMessage();
} }
await this.options.channel.setTyping?.(this.options.chatJid, false); await this.setTyping(false, 'turn:handle-output', {
outputStatus: result.status,
phase,
});
if (result.status === 'success' && !this.poisonedSessionDetected) { if (result.status === 'success' && !this.poisonedSessionDetected) {
this.requestAgentClose('output-delivered-close'); this.requestAgentClose('output-delivered-close');
} }
@@ -288,7 +311,7 @@ export class MessageTurnController {
deliverySucceeded: boolean; deliverySucceeded: boolean;
visiblePhase: VisiblePhase; visiblePhase: VisiblePhase;
}> { }> {
await this.options.channel.setTyping?.(this.options.chatJid, false); await this.setTyping(false, 'turn:finish', { outputStatus });
if (outputStatus === 'error') { if (outputStatus === 'error') {
this.hadError = true; this.hadError = true;

View File

@@ -142,6 +142,22 @@ function clearUsageAvailabilityCache(): void {
usageAvailabilityCheckPromise = null; usageAvailabilityCheckPromise = null;
} }
function logCooldownTransition(
level: 'debug' | 'info' | 'warn',
transition: string,
fields: Record<string, unknown>,
message: string,
): void {
logger[level](
{
transition,
provider: 'claude',
...fields,
},
message,
);
}
async function getClaudeUsageAvailability(): Promise< async function getClaudeUsageAvailability(): Promise<
'available' | 'exhausted' | 'unknown' 'available' | 'exhausted' | 'unknown'
> { > {
@@ -187,10 +203,12 @@ export async function getActiveProvider(): Promise<string> {
if (cooldown.reason === 'usage-exhausted') { if (cooldown.reason === 'usage-exhausted') {
const usageAvailability = await getClaudeUsageAvailability(); const usageAvailability = await getClaudeUsageAvailability();
if (usageAvailability === 'available') { if (usageAvailability === 'available') {
logger.info( logCooldownTransition(
'info',
'cooldown:recover',
{ {
provider: 'claude',
reason: cooldown.reason, reason: cooldown.reason,
fallbackProvider: config.providerName,
}, },
'Claude usage recovered, retrying primary provider', 'Claude usage recovered, retrying primary provider',
); );
@@ -204,17 +222,27 @@ export async function getActiveProvider(): Promise<string> {
getTokenCount() > 1 && getTokenCount() > 1 &&
rotateToken(undefined, { ignoreRateLimits: true }) rotateToken(undefined, { ignoreRateLimits: true })
) { ) {
logger.info( logCooldownTransition(
'info',
'cooldown:recover',
{
reason: cooldown.reason,
fallbackProvider: config.providerName,
recovery: 'token-rotation',
},
'Claude current token exhausted, rotated to next token — retrying', 'Claude current token exhausted, rotated to next token — retrying',
); );
cooldown = null; cooldown = null;
clearUsageAvailabilityCache(); clearUsageAvailabilityCache();
return 'claude'; return 'claude';
} }
logger.debug( logCooldownTransition(
'debug',
'cooldown:stay',
{ {
provider: config.providerName,
reason: cooldown.reason, reason: cooldown.reason,
fallbackProvider: config.providerName,
usageAvailability,
}, },
'All Claude tokens exhausted, keeping cooldown active', 'All Claude tokens exhausted, keeping cooldown active',
); );
@@ -223,14 +251,26 @@ export async function getActiveProvider(): Promise<string> {
} }
if (Date.now() < cooldown.expiresAt) { if (Date.now() < cooldown.expiresAt) {
logCooldownTransition(
'debug',
'cooldown:stay',
{
reason: cooldown.reason,
fallbackProvider: config.providerName,
remainingMs: cooldown.expiresAt - Date.now(),
},
'Claude cooldown still active, routing to fallback provider',
);
return config.providerName; return config.providerName;
} }
// Cooldown expired — try Claude again // Cooldown expired — try Claude again
logger.info( logCooldownTransition(
'info',
'cooldown:expire',
{ {
provider: 'claude',
cooldownDurationMs: cooldown.expiresAt - cooldown.startedAt, cooldownDurationMs: cooldown.expiresAt - cooldown.startedAt,
reason: cooldown.reason, reason: cooldown.reason,
fallbackProvider: config.providerName,
}, },
'Claude cooldown expired, retrying primary provider', 'Claude cooldown expired, retrying primary provider',
); );
@@ -259,7 +299,9 @@ export function markPrimaryCooldown(
}; };
clearUsageAvailabilityCache(); clearUsageAvailabilityCache();
logger.info( logCooldownTransition(
'info',
'cooldown:enter',
{ {
reason, reason,
cooldownMs: durationMs, cooldownMs: durationMs,
@@ -274,7 +316,9 @@ export function markPrimaryCooldown(
export function clearPrimaryCooldown(): void { export function clearPrimaryCooldown(): void {
clearUsageAvailabilityCache(); clearUsageAvailabilityCache();
if (cooldown) { if (cooldown) {
logger.info( logCooldownTransition(
'info',
'cooldown:clear',
{ reason: cooldown.reason }, { reason: cooldown.reason },
'Claude cooldown cleared manually', 'Claude cooldown cleared manually',
); );

View File

@@ -118,7 +118,9 @@ export function rotateToken(
): boolean { ): boolean {
if (tokens.length <= 1) return false; if (tokens.length <= 1) return false;
tokens[currentIndex].rateLimitedUntil = computeCooldownUntil(errorMessage); const previousIndex = currentIndex;
const cooldownUntil = computeCooldownUntil(errorMessage);
tokens[currentIndex].rateLimitedUntil = cooldownUntil;
const nextIdx = findNextAvailable(tokens, currentIndex, opts); const nextIdx = findNextAvailable(tokens, currentIndex, opts);
if (nextIdx !== null) { if (nextIdx !== null) {
@@ -126,9 +128,14 @@ export function rotateToken(
currentIndex = nextIdx; currentIndex = nextIdx;
logger.info( logger.info(
{ {
tokenIndex: currentIndex, transition: 'rotation:execute',
fromIndex: previousIndex,
toIndex: currentIndex,
totalTokens: tokens.length, totalTokens: tokens.length,
ignoreRL: opts?.ignoreRateLimits ?? false, ignoreRL: opts?.ignoreRateLimits ?? false,
cooldownUntil:
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
reason: errorMessage ?? null,
}, },
`Rotated to token #${currentIndex + 1}/${tokens.length}`, `Rotated to token #${currentIndex + 1}/${tokens.length}`,
); );
@@ -137,7 +144,15 @@ export function rotateToken(
} }
logger.warn( logger.warn(
{ totalTokens: tokens.length }, {
transition: 'rotation:skip',
fromIndex: previousIndex,
totalTokens: tokens.length,
ignoreRL: opts?.ignoreRateLimits ?? false,
cooldownUntil:
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
reason: errorMessage ?? null,
},
'All tokens are rate-limited, falling through to provider fallback', 'All tokens are rate-limited, falling through to provider fallback',
); );
return false; return false;
@@ -148,7 +163,16 @@ export function markTokenHealthy(): void {
if (tokens.length === 0) return; if (tokens.length === 0) return;
const state = tokens[currentIndex]; const state = tokens[currentIndex];
if (state?.rateLimitedUntil) { if (state?.rateLimitedUntil) {
const previousCooldownUntil = state.rateLimitedUntil;
state.rateLimitedUntil = null; state.rateLimitedUntil = null;
logger.info(
{
transition: 'rotation:clear-rate-limit',
tokenIndex: currentIndex,
cooldownUntil: new Date(previousCooldownUntil).toISOString(),
},
'Cleared Claude token rate-limit state after successful response',
);
saveState(); saveState();
} }
} }