From 6c6b67dacadc349289f41201d868a2f45724d1e1 Mon Sep 17 00:00:00 2001 From: ejclaw Date: Wed, 29 Apr 2026 22:20:06 +0900 Subject: [PATCH] detect owner codex bad request loops --- src/codex-bad-request-signal.ts | 7 + ...b-paired-turn-attempts-bad-request.test.ts | 155 ++++++++++++++++ src/db.ts | 1 + src/db/paired-turn-attempts.ts | 61 +++++++ src/db/runtime-paired.ts | 12 ++ src/message-runtime-turns.ts | 13 ++ src/message-runtime.test.ts | 1 + src/session-auto-healer.test.ts | 166 ++++++++++++++++++ src/session-auto-healer.ts | 97 ++++++++++ src/session-recovery.test.ts | 21 +++ src/session-recovery.ts | 11 ++ 11 files changed, 545 insertions(+) create mode 100644 src/codex-bad-request-signal.ts create mode 100644 src/db-paired-turn-attempts-bad-request.test.ts create mode 100644 src/session-auto-healer.test.ts create mode 100644 src/session-auto-healer.ts diff --git a/src/codex-bad-request-signal.ts b/src/codex-bad-request-signal.ts new file mode 100644 index 0000000..99ae401 --- /dev/null +++ b/src/codex-bad-request-signal.ts @@ -0,0 +1,7 @@ +export const CODEX_BAD_REQUEST_DETAIL_JSON = '{"detail":"Bad Request"}'; + +export function isCodexBadRequestText( + text: string | null | undefined, +): boolean { + return text?.trim() === CODEX_BAD_REQUEST_DETAIL_JSON; +} diff --git a/src/db-paired-turn-attempts-bad-request.test.ts b/src/db-paired-turn-attempts-bad-request.test.ts new file mode 100644 index 0000000..48467dc --- /dev/null +++ b/src/db-paired-turn-attempts-bad-request.test.ts @@ -0,0 +1,155 @@ +import { beforeEach, describe, expect, it } from 'vitest'; + +import { + _initTestDatabase, + createPairedTask, + failPairedTurn, + getOwnerCodexBadRequestFailureSummaryForTask, + markPairedTurnRunning, +} from './db.js'; +import { CODEX_MAIN_SERVICE_ID } from './config.js'; +import { buildPairedTurnIdentity } from './paired-turn-identity.js'; +import type { PairedTask } from './types.js'; + +const BAD_REQUEST_ERROR = '{"detail":"Bad Request"}'; + +beforeEach(() => { + _initTestDatabase(); +}); + +function makeTask(overrides: Partial = {}): PairedTask { + return { + id: 'task-bad-request', + chat_jid: 'dc:room', + group_folder: 'eyejokerdb-9', + owner_service_id: CODEX_MAIN_SERVICE_ID, + reviewer_service_id: 'claude', + title: null, + source_ref: null, + plan_notes: null, + review_requested_at: null, + round_trip_count: 1, + status: 'active', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-04-29T08:00:00.000Z', + updated_at: '2026-04-29T08:00:00.000Z', + ...overrides, + }; +} + +function recordOwnerCodexFailure(args: { + taskId: string; + taskUpdatedAt: string; + runId: string; + error: string; +}): void { + const turnIdentity = buildPairedTurnIdentity({ + taskId: args.taskId, + taskUpdatedAt: args.taskUpdatedAt, + intentKind: 'owner-follow-up', + role: 'owner', + }); + + markPairedTurnRunning({ + turnIdentity, + executorServiceId: CODEX_MAIN_SERVICE_ID, + executorAgentType: 'codex', + runId: args.runId, + }); + failPairedTurn({ turnIdentity, error: args.error }); +} + +describe('owner Codex Bad Request attempt summaries', () => { + it('summarizes consecutive owner Codex Bad Request failures for a task', () => { + const task = makeTask(); + createPairedTask(task); + + for (const runId of ['run-1', 'run-2', 'run-3']) { + recordOwnerCodexFailure({ + taskId: task.id, + taskUpdatedAt: task.updated_at, + runId, + error: BAD_REQUEST_ERROR, + }); + } + + expect( + getOwnerCodexBadRequestFailureSummaryForTask({ + taskId: task.id, + threshold: 3, + }), + ).toMatchObject({ + taskId: task.id, + failures: 3, + }); + }); + + it('does not summarize when the latest owner Codex failures are not all the narrow Bad Request signal', () => { + const task = makeTask(); + createPairedTask(task); + + recordOwnerCodexFailure({ + taskId: task.id, + taskUpdatedAt: task.updated_at, + runId: 'run-1', + error: BAD_REQUEST_ERROR, + }); + recordOwnerCodexFailure({ + taskId: task.id, + taskUpdatedAt: task.updated_at, + runId: 'run-2', + error: BAD_REQUEST_ERROR, + }); + recordOwnerCodexFailure({ + taskId: task.id, + taskUpdatedAt: task.updated_at, + runId: 'run-3', + error: 'HTTP 400 Bad Request', + }); + + expect( + getOwnerCodexBadRequestFailureSummaryForTask({ + taskId: task.id, + threshold: 3, + }), + ).toBeNull(); + }); + + it('counts only the latest consecutive Bad Request failures', () => { + const task = makeTask(); + createPairedTask(task); + + recordOwnerCodexFailure({ + taskId: task.id, + taskUpdatedAt: task.updated_at, + runId: 'run-1', + error: BAD_REQUEST_ERROR, + }); + recordOwnerCodexFailure({ + taskId: task.id, + taskUpdatedAt: task.updated_at, + runId: 'run-2', + error: 'HTTP 400 Bad Request', + }); + for (const runId of ['run-3', 'run-4', 'run-5']) { + recordOwnerCodexFailure({ + taskId: task.id, + taskUpdatedAt: task.updated_at, + runId, + error: BAD_REQUEST_ERROR, + }); + } + + expect( + getOwnerCodexBadRequestFailureSummaryForTask({ + taskId: task.id, + threshold: 3, + }), + ).toMatchObject({ + taskId: task.id, + failures: 3, + }); + }); +}); diff --git a/src/db.ts b/src/db.ts index 34bf8a4..83ca622 100644 --- a/src/db.ts +++ b/src/db.ts @@ -98,6 +98,7 @@ export { getLatestPairedTaskForChat, getLatestPreviousPairedTaskForChat, getLatestTurnNumber, + getOwnerCodexBadRequestFailureSummaryForTask, getPairedProject, getPairedTaskById, getPairedTurnAttempts, diff --git a/src/db/paired-turn-attempts.ts b/src/db/paired-turn-attempts.ts index 3b24b9d..d32a4eb 100644 --- a/src/db/paired-turn-attempts.ts +++ b/src/db/paired-turn-attempts.ts @@ -1,6 +1,7 @@ import { Database } from 'bun:sqlite'; import { normalizeServiceId } from '../config.js'; +import { CODEX_BAD_REQUEST_DETAIL_JSON } from '../codex-bad-request-signal.js'; import type { PairedTurnIdentity } from '../paired-turn-identity.js'; import { inferAgentTypeFromServiceShadow } from '../role-service-shadow.js'; import type { AgentType, PairedRoomRole } from '../types.js'; @@ -33,6 +34,13 @@ export interface PairedTurnAttemptRecord { last_error: string | null; } +export interface OwnerCodexBadRequestFailureSummary { + taskId: string; + failures: number; + firstFailureAt: string; + latestFailureAt: string; +} + function resolveExecutorMetadata(args: { executorServiceId?: string | null; executorAgentType?: AgentType | null; @@ -423,6 +431,59 @@ export function getPairedTurnAttemptIdFromDatabase( return row?.attempt_id ?? null; } +export function getOwnerCodexBadRequestFailureSummaryForTaskFromDatabase( + database: Database, + args: { + taskId: string; + threshold: number; + }, +): OwnerCodexBadRequestFailureSummary | null { + const threshold = Math.max(1, Math.floor(args.threshold)); + const attempts = database + .prepare( + ` + SELECT state, last_error, created_at + FROM paired_turn_attempts + WHERE task_id = ? + AND role = 'owner' + AND executor_agent_type = 'codex' + ORDER BY created_at DESC, attempt_no DESC + `, + ) + .all(args.taskId) as Array<{ + state: PairedTurnAttemptState; + last_error: string | null; + created_at: string; + }>; + + const consecutiveFailures = []; + for (const attempt of attempts) { + if ( + attempt.state === 'failed' && + attempt.last_error?.trim() === CODEX_BAD_REQUEST_DETAIL_JSON + ) { + consecutiveFailures.push(attempt); + continue; + } + break; + } + + if (consecutiveFailures.length < threshold) { + return null; + } + + const firstFailureAt = + consecutiveFailures[consecutiveFailures.length - 1].created_at; + const latestFailureAt = consecutiveFailures[0].created_at; + + return { + taskId: args.taskId, + failures: consecutiveFailures.length, + firstFailureAt, + latestFailureAt, + }; +} + export function setPairedTurnAttemptContinuationHandoffIdInDatabase( database: Database, args: { diff --git a/src/db/runtime-paired.ts b/src/db/runtime-paired.ts index e6c76b5..30049e0 100644 --- a/src/db/runtime-paired.ts +++ b/src/db/runtime-paired.ts @@ -43,7 +43,9 @@ import { } from './paired-state.js'; import { clearPairedTurnAttemptsInDatabase, + getOwnerCodexBadRequestFailureSummaryForTaskFromDatabase, getPairedTurnAttemptsForTurnFromDatabase, + type OwnerCodexBadRequestFailureSummary, type PairedTurnAttemptRecord, } from './paired-turn-attempts.js'; import { @@ -237,6 +239,16 @@ export function getPairedTurnAttempts( return getPairedTurnAttemptsForTurnFromDatabase(requireDatabase(), turnId); } +export function getOwnerCodexBadRequestFailureSummaryForTask(args: { + taskId: string; + threshold: number; +}): OwnerCodexBadRequestFailureSummary | null { + return getOwnerCodexBadRequestFailureSummaryForTaskFromDatabase( + requireDatabase(), + args, + ); +} + export function upsertPairedWorkspace(workspace: PairedWorkspace): void { upsertPairedWorkspaceInDatabase(requireDatabase(), workspace); } diff --git a/src/message-runtime-turns.ts b/src/message-runtime-turns.ts index 7b4eae1..97e87c8 100644 --- a/src/message-runtime-turns.ts +++ b/src/message-runtime-turns.ts @@ -9,6 +9,7 @@ import { } from './service-routing.js'; import { resolvePairedTurnRunOwnership } from './paired-turn-run-ownership.js'; import { normalizeMessageForDedupe } from './router.js'; +import { notifyOwnerCodexBadRequestObservation } from './session-auto-healer.js'; import type { ExecuteTurnFn } from './message-runtime-types.js'; import type { AgentType, @@ -255,6 +256,18 @@ export function createExecuteTurn(deps: CreateExecuteTurnDeps): ExecuteTurnFn { const { deliverySucceeded, visiblePhase } = await turnController.finish(outputStatus); + await notifyOwnerCodexBadRequestObservation({ + chatJid, + runId, + groupFolder: group.folder, + channel, + outputStatus, + visiblePhase, + deliveryRole: resolvedDeliveryRole, + agentType: args.forcedAgentType ?? group.agentType ?? 'claude-code', + pairedTurnIdentity: args.pairedTurnIdentity ?? null, + }); + if (deliverySucceeded) { await deps.afterDeliverySuccess?.({ chatJid, diff --git a/src/message-runtime.test.ts b/src/message-runtime.test.ts index cc5f0d7..ee306fd 100644 --- a/src/message-runtime.test.ts +++ b/src/message-runtime.test.ts @@ -165,6 +165,7 @@ vi.mock('./db.js', () => { getLatestPreviousPairedTaskForChat: vi.fn(() => undefined), getPairedTaskById: vi.fn(() => undefined), getPairedTurnOutputs: vi.fn(() => []), + getOwnerCodexBadRequestFailureSummaryForTask: vi.fn(() => null), getRecentChatMessages: vi.fn(() => []), createProducedWorkItem: vi.fn((input) => ({ id: 1, diff --git a/src/session-auto-healer.test.ts b/src/session-auto-healer.test.ts new file mode 100644 index 0000000..00d5a2d --- /dev/null +++ b/src/session-auto-healer.test.ts @@ -0,0 +1,166 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + _initTestDatabase, + createPairedTask, + failPairedTurn, + markPairedTurnRunning, +} from './db.js'; +import { CODEX_MAIN_SERVICE_ID } from './config.js'; +import { + getCodexBadRequestRepeatThreshold, + notifyOwnerCodexBadRequestObservation, +} from './session-auto-healer.js'; +import type { PairedTask } from './types.js'; + +const BAD_REQUEST_ERROR = '{"detail":"Bad Request"}'; +const originalThresholdEnv = process.env.CODEX_BAD_REQUEST_REPEAT_THRESHOLD; + +const pairedTurnIdentity = { + turnId: 'task-1:updated:owner-follow-up', + taskId: 'task-1', + taskUpdatedAt: '2026-04-29T01:00:00.000Z', + intentKind: 'owner-follow-up' as const, + role: 'owner' as const, +}; + +function makeTask(overrides: Partial = {}): PairedTask { + return { + id: 'task-1', + chat_jid: 'dc:room', + group_folder: 'eyejokerdb-9', + owner_service_id: CODEX_MAIN_SERVICE_ID, + reviewer_service_id: 'claude', + title: null, + source_ref: null, + plan_notes: null, + review_requested_at: null, + round_trip_count: 1, + status: 'active', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-04-29T08:00:00.000Z', + updated_at: pairedTurnIdentity.taskUpdatedAt, + ...overrides, + }; +} + +function recordOwnerCodexFailure(runId: string): void { + markPairedTurnRunning({ + turnIdentity: pairedTurnIdentity, + executorServiceId: CODEX_MAIN_SERVICE_ID, + executorAgentType: 'codex', + runId, + }); + failPairedTurn({ + turnIdentity: pairedTurnIdentity, + error: BAD_REQUEST_ERROR, + }); +} + +describe('session-auto-healer', () => { + beforeEach(() => { + _initTestDatabase(); + delete process.env.CODEX_BAD_REQUEST_REPEAT_THRESHOLD; + }); + + afterEach(() => { + if (originalThresholdEnv === undefined) { + delete process.env.CODEX_BAD_REQUEST_REPEAT_THRESHOLD; + } else { + process.env.CODEX_BAD_REQUEST_REPEAT_THRESHOLD = originalThresholdEnv; + } + }); + + it('uses a conservative default Bad Request repeat threshold', () => { + expect(getCodexBadRequestRepeatThreshold()).toBe(3); + }); + + it('supports threshold override for observation tuning', () => { + process.env.CODEX_BAD_REQUEST_REPEAT_THRESHOLD = '2'; + + expect(getCodexBadRequestRepeatThreshold()).toBe(2); + }); + + it('ignores invalid threshold overrides', () => { + process.env.CODEX_BAD_REQUEST_REPEAT_THRESHOLD = '1'; + + expect(getCodexBadRequestRepeatThreshold()).toBe(3); + }); + + it('notifies once when owner Codex reaches the Bad Request observation threshold', async () => { + createPairedTask(makeTask()); + for (const runId of ['run-1', 'run-2', 'run-3']) { + recordOwnerCodexFailure(runId); + } + const channel = { sendMessage: vi.fn() }; + + const notified = await notifyOwnerCodexBadRequestObservation({ + chatJid: 'dc:room', + runId: 'run-1', + groupFolder: 'eyejokerdb-9', + channel, + outputStatus: 'error', + visiblePhase: 'silent', + deliveryRole: 'owner', + agentType: 'codex', + pairedTurnIdentity, + threshold: 3, + }); + + expect(notified).toBe(true); + expect(channel.sendMessage).toHaveBeenCalledWith( + 'dc:room', + expect.stringContaining('자동복구는 아직 비활성화'), + ); + expect(channel.sendMessage).toHaveBeenCalledWith( + 'dc:room', + expect.stringContaining('task=task-1'), + ); + }); + + it('does not notify when the failure had visible output', async () => { + const channel = { sendMessage: vi.fn() }; + + const notified = await notifyOwnerCodexBadRequestObservation({ + chatJid: 'dc:room', + runId: 'run-1', + groupFolder: 'eyejokerdb-9', + channel, + outputStatus: 'error', + visiblePhase: 'progress', + deliveryRole: 'owner', + agentType: 'codex', + pairedTurnIdentity, + threshold: 3, + }); + + expect(notified).toBe(false); + expect(channel.sendMessage).not.toHaveBeenCalled(); + }); + + it('does not notify again after the exact threshold has already passed', async () => { + createPairedTask(makeTask()); + for (const runId of ['run-1', 'run-2', 'run-3', 'run-4']) { + recordOwnerCodexFailure(runId); + } + const channel = { sendMessage: vi.fn() }; + + const notified = await notifyOwnerCodexBadRequestObservation({ + chatJid: 'dc:room', + runId: 'run-1', + groupFolder: 'eyejokerdb-9', + channel, + outputStatus: 'error', + visiblePhase: 'silent', + deliveryRole: 'owner', + agentType: 'codex', + pairedTurnIdentity, + threshold: 3, + }); + + expect(notified).toBe(false); + expect(channel.sendMessage).not.toHaveBeenCalled(); + }); +}); diff --git a/src/session-auto-healer.ts b/src/session-auto-healer.ts new file mode 100644 index 0000000..ef87d77 --- /dev/null +++ b/src/session-auto-healer.ts @@ -0,0 +1,97 @@ +import { getOwnerCodexBadRequestFailureSummaryForTask } from './db.js'; +import { getEnv } from './env.js'; +import { logger } from './logger.js'; +import type { PairedTurnIdentity } from './paired-turn-identity.js'; +import type { + AgentType, + Channel, + PairedRoomRole, + VisiblePhase, +} from './types.js'; + +const DEFAULT_CODEX_BAD_REQUEST_REPEAT_THRESHOLD = 3; + +export function getCodexBadRequestRepeatThreshold(): number { + const raw = getEnv('CODEX_BAD_REQUEST_REPEAT_THRESHOLD'); + if (!raw) { + return DEFAULT_CODEX_BAD_REQUEST_REPEAT_THRESHOLD; + } + + const parsed = Number.parseInt(raw, 10); + if (!Number.isFinite(parsed) || parsed < 2) { + return DEFAULT_CODEX_BAD_REQUEST_REPEAT_THRESHOLD; + } + return parsed; +} + +export async function notifyOwnerCodexBadRequestObservation(args: { + chatJid: string; + runId: string; + groupFolder: string; + channel: Pick; + outputStatus: 'success' | 'error'; + visiblePhase: VisiblePhase; + deliveryRole: PairedRoomRole | null; + agentType: AgentType; + pairedTurnIdentity: PairedTurnIdentity | null; + threshold?: number; +}): Promise { + if ( + args.outputStatus !== 'error' || + args.visiblePhase !== 'silent' || + args.agentType !== 'codex' + ) { + return false; + } + + const role = args.pairedTurnIdentity?.role ?? args.deliveryRole; + if (role !== 'owner' || !args.pairedTurnIdentity) { + return false; + } + + const threshold = args.threshold ?? getCodexBadRequestRepeatThreshold(); + const summary = getOwnerCodexBadRequestFailureSummaryForTask({ + taskId: args.pairedTurnIdentity.taskId, + threshold, + }); + if (!summary || summary.failures !== threshold) { + return false; + } + + const message = [ + `🟡 owner Codex 세션이 {"detail":"Bad Request"}로 ${summary.failures}회 연속 무출력 실패했습니다.`, + '자동복구는 아직 비활성화되어 있습니다. 현재는 감지만 수행하며, 필요하면 `/clear`로 세션을 초기화해 주세요.', + `task=${summary.taskId}`, + `latest=${summary.latestFailureAt}`, + ].join('\n'); + + try { + await args.channel.sendMessage(args.chatJid, message); + } catch (err) { + logger.warn( + { + err, + chatJid: args.chatJid, + runId: args.runId, + groupFolder: args.groupFolder, + taskId: summary.taskId, + }, + 'Failed to send owner Codex Bad Request observation notice', + ); + return false; + } + + logger.warn( + { + chatJid: args.chatJid, + runId: args.runId, + groupFolder: args.groupFolder, + taskId: summary.taskId, + failures: summary.failures, + firstFailureAt: summary.firstFailureAt, + latestFailureAt: summary.latestFailureAt, + }, + 'Detected repeated owner Codex Bad Request failures without visible output', + ); + return true; +} diff --git a/src/session-recovery.test.ts b/src/session-recovery.test.ts index c4ba31d..453af1d 100644 --- a/src/session-recovery.test.ts +++ b/src/session-recovery.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import { + isCodexBadRequestSignal, shouldResetCodexSessionOnAgentFailure, shouldResetSessionOnAgentFailure, shouldRetryFreshCodexSessionOnAgentFailure, @@ -133,4 +134,24 @@ describe('shouldRetryFreshCodexSessionOnAgentFailure', () => { }), ).toBe(true); }); + + it('does not retry generic Codex Bad Request signals during observation-only rollout', () => { + const output = { + result: null, + error: '{"detail":"Bad Request"}', + }; + + expect(isCodexBadRequestSignal(output)).toBe(true); + expect(shouldResetCodexSessionOnAgentFailure(output)).toBe(false); + expect(shouldRetryFreshCodexSessionOnAgentFailure(output)).toBe(false); + }); + + it('does not flag unrelated Bad Request text as the narrow Codex signal', () => { + expect( + isCodexBadRequestSignal({ + result: null, + error: 'HTTP 400 Bad Request', + }), + ).toBe(false); + }); }); diff --git a/src/session-recovery.ts b/src/session-recovery.ts index 6ce675a..b5a6486 100644 --- a/src/session-recovery.ts +++ b/src/session-recovery.ts @@ -1,5 +1,6 @@ import { getAgentOutputText } from './agent-output.js'; import type { AgentOutput } from './agent-runner.js'; +import { isCodexBadRequestText } from './codex-bad-request-signal.js'; const SESSION_RESET_PATTERNS = [ /An image in the conversation exceeds the dimension limit for many-image requests \(2000px\)\./i, @@ -74,3 +75,13 @@ export function shouldRetryFreshCodexSessionOnAgentFailure( ): boolean { return shouldResetCodexSessionOnAgentFailure(output); } + +export function isCodexBadRequestSignal( + output: Pick, +): boolean { + const texts = [ + ...toText(getAgentOutputText(output)), + ...toText(output.error), + ]; + return texts.some((text) => isCodexBadRequestText(text)); +}