Merge pull request #109 from phj1081/codex/owner/ejclaw

Detect owner Codex Bad Request loops

src/codex-bad-request-signal.ts

@@ -0,0 +1,7 @@
+export const CODEX_BAD_REQUEST_DETAIL_JSON = '{"detail":"Bad Request"}';
+
+export function isCodexBadRequestText(
+  text: string | null | undefined,
+): boolean {
+  return text?.trim() === CODEX_BAD_REQUEST_DETAIL_JSON;
+}

src/db-paired-turn-attempts-bad-request.test.ts

@@ -0,0 +1,155 @@
+import { beforeEach, describe, expect, it } from 'vitest';
+
+import {
+  _initTestDatabase,
+  createPairedTask,
+  failPairedTurn,
+  getOwnerCodexBadRequestFailureSummaryForTask,
+  markPairedTurnRunning,
+} from './db.js';
+import { CODEX_MAIN_SERVICE_ID } from './config.js';
+import { buildPairedTurnIdentity } from './paired-turn-identity.js';
+import type { PairedTask } from './types.js';
+
+const BAD_REQUEST_ERROR = '{"detail":"Bad Request"}';
+
+beforeEach(() => {
+  _initTestDatabase();
+});
+
+function makeTask(overrides: Partial<PairedTask> = {}): PairedTask {
+  return {
+    id: 'task-bad-request',
+    chat_jid: 'dc:room',
+    group_folder: 'eyejokerdb-9',
+    owner_service_id: CODEX_MAIN_SERVICE_ID,
+    reviewer_service_id: 'claude',
+    title: null,
+    source_ref: null,
+    plan_notes: null,
+    review_requested_at: null,
+    round_trip_count: 1,
+    status: 'active',
+    arbiter_verdict: null,
+    arbiter_requested_at: null,
+    completion_reason: null,
+    created_at: '2026-04-29T08:00:00.000Z',
+    updated_at: '2026-04-29T08:00:00.000Z',
+    ...overrides,
+  };
+}
+
+function recordOwnerCodexFailure(args: {
+  taskId: string;
+  taskUpdatedAt: string;
+  runId: string;
+  error: string;
+}): void {
+  const turnIdentity = buildPairedTurnIdentity({
+    taskId: args.taskId,
+    taskUpdatedAt: args.taskUpdatedAt,
+    intentKind: 'owner-follow-up',
+    role: 'owner',
+  });
+
+  markPairedTurnRunning({
+    turnIdentity,
+    executorServiceId: CODEX_MAIN_SERVICE_ID,
+    executorAgentType: 'codex',
+    runId: args.runId,
+  });
+  failPairedTurn({ turnIdentity, error: args.error });
+}
+
+describe('owner Codex Bad Request attempt summaries', () => {
+  it('summarizes consecutive owner Codex Bad Request failures for a task', () => {
+    const task = makeTask();
+    createPairedTask(task);
+
+    for (const runId of ['run-1', 'run-2', 'run-3']) {
+      recordOwnerCodexFailure({
+        taskId: task.id,
+        taskUpdatedAt: task.updated_at,
+        runId,
+        error: BAD_REQUEST_ERROR,
+      });
+    }
+
+    expect(
+      getOwnerCodexBadRequestFailureSummaryForTask({
+        taskId: task.id,
+        threshold: 3,
+      }),
+    ).toMatchObject({
+      taskId: task.id,
+      failures: 3,
+    });
+  });
+
+  it('does not summarize when the latest owner Codex failures are not all the narrow Bad Request signal', () => {
+    const task = makeTask();
+    createPairedTask(task);
+
+    recordOwnerCodexFailure({
+      taskId: task.id,
+      taskUpdatedAt: task.updated_at,
+      runId: 'run-1',
+      error: BAD_REQUEST_ERROR,
+    });
+    recordOwnerCodexFailure({
+      taskId: task.id,
+      taskUpdatedAt: task.updated_at,
+      runId: 'run-2',
+      error: BAD_REQUEST_ERROR,
+    });
+    recordOwnerCodexFailure({
+      taskId: task.id,
+      taskUpdatedAt: task.updated_at,
+      runId: 'run-3',
+      error: 'HTTP 400 Bad Request',
+    });
+
+    expect(
+      getOwnerCodexBadRequestFailureSummaryForTask({
+        taskId: task.id,
+        threshold: 3,
+      }),
+    ).toBeNull();
+  });
+
+  it('counts only the latest consecutive Bad Request failures', () => {
+    const task = makeTask();
+    createPairedTask(task);
+
+    recordOwnerCodexFailure({
+      taskId: task.id,
+      taskUpdatedAt: task.updated_at,
+      runId: 'run-1',
+      error: BAD_REQUEST_ERROR,
+    });
+    recordOwnerCodexFailure({
+      taskId: task.id,
+      taskUpdatedAt: task.updated_at,
+      runId: 'run-2',
+      error: 'HTTP 400 Bad Request',
+    });
+    for (const runId of ['run-3', 'run-4', 'run-5']) {
+      recordOwnerCodexFailure({
+        taskId: task.id,
+        taskUpdatedAt: task.updated_at,
+        runId,
+        error: BAD_REQUEST_ERROR,
+      });
+    }
+
+    expect(
+      getOwnerCodexBadRequestFailureSummaryForTask({
+        taskId: task.id,
+        threshold: 3,
+      }),
+    ).toMatchObject({
+      taskId: task.id,
+      failures: 3,
+    });
+  });
+});

src/db.ts

@@ -98,6 +98,7 @@ export {
   getLatestPairedTaskForChat,
   getLatestPreviousPairedTaskForChat,
   getLatestTurnNumber,
+  getOwnerCodexBadRequestFailureSummaryForTask,
   getPairedProject,
   getPairedTaskById,
   getPairedTurnAttempts,

src/db/paired-turn-attempts.ts

@@ -1,6 +1,7 @@
 import { Database } from 'bun:sqlite';
 
 import { normalizeServiceId } from '../config.js';
+import { CODEX_BAD_REQUEST_DETAIL_JSON } from '../codex-bad-request-signal.js';
 import type { PairedTurnIdentity } from '../paired-turn-identity.js';
 import { inferAgentTypeFromServiceShadow } from '../role-service-shadow.js';
 import type { AgentType, PairedRoomRole } from '../types.js';
@@ -33,6 +34,13 @@ export interface PairedTurnAttemptRecord {
   last_error: string | null;
 }
 
+export interface OwnerCodexBadRequestFailureSummary {
+  taskId: string;
+  failures: number;
+  firstFailureAt: string;
+  latestFailureAt: string;
+}
+
 function resolveExecutorMetadata(args: {
   executorServiceId?: string | null;
   executorAgentType?: AgentType | null;
@@ -423,6 +431,59 @@ export function getPairedTurnAttemptIdFromDatabase(
   return row?.attempt_id ?? null;
 }
 
+export function getOwnerCodexBadRequestFailureSummaryForTaskFromDatabase(
+  database: Database,
+  args: {
+    taskId: string;
+    threshold: number;
+  },
+): OwnerCodexBadRequestFailureSummary | null {
+  const threshold = Math.max(1, Math.floor(args.threshold));
+  const attempts = database
+    .prepare(
+      `
+        SELECT state, last_error, created_at
+          FROM paired_turn_attempts
+         WHERE task_id = ?
+           AND role = 'owner'
+           AND executor_agent_type = 'codex'
+         ORDER BY created_at DESC, attempt_no DESC
+      `,
+    )
+    .all(args.taskId) as Array<{
+    state: PairedTurnAttemptState;
+    last_error: string | null;
+    created_at: string;
+  }>;
+
+  const consecutiveFailures = [];
+  for (const attempt of attempts) {
+    if (
+      attempt.state === 'failed' &&
+      attempt.last_error?.trim() === CODEX_BAD_REQUEST_DETAIL_JSON
+    ) {
+      consecutiveFailures.push(attempt);
+      continue;
+    }
+    break;
+  }
+
+  if (consecutiveFailures.length < threshold) {
+    return null;
+  }
+
+  const firstFailureAt =
+    consecutiveFailures[consecutiveFailures.length - 1].created_at;
+  const latestFailureAt = consecutiveFailures[0].created_at;
+
+  return {
+    taskId: args.taskId,
+    failures: consecutiveFailures.length,
+    firstFailureAt,
+    latestFailureAt,
+  };
+}
+
 export function setPairedTurnAttemptContinuationHandoffIdInDatabase(
   database: Database,
   args: {

src/db/runtime-paired.ts

@@ -43,7 +43,9 @@ import {
 } from './paired-state.js';
 import {
   clearPairedTurnAttemptsInDatabase,
+  getOwnerCodexBadRequestFailureSummaryForTaskFromDatabase,
   getPairedTurnAttemptsForTurnFromDatabase,
+  type OwnerCodexBadRequestFailureSummary,
   type PairedTurnAttemptRecord,
 } from './paired-turn-attempts.js';
 import {
@@ -237,6 +239,16 @@ export function getPairedTurnAttempts(
   return getPairedTurnAttemptsForTurnFromDatabase(requireDatabase(), turnId);
 }
 
+export function getOwnerCodexBadRequestFailureSummaryForTask(args: {
+  taskId: string;
+  threshold: number;
+}): OwnerCodexBadRequestFailureSummary | null {
+  return getOwnerCodexBadRequestFailureSummaryForTaskFromDatabase(
+    requireDatabase(),
+    args,
+  );
+}
+
 export function upsertPairedWorkspace(workspace: PairedWorkspace): void {
   upsertPairedWorkspaceInDatabase(requireDatabase(), workspace);
 }

src/message-runtime-turns.ts

@@ -9,6 +9,7 @@ import {
 } from './service-routing.js';
 import { resolvePairedTurnRunOwnership } from './paired-turn-run-ownership.js';
 import { normalizeMessageForDedupe } from './router.js';
+import { notifyOwnerCodexBadRequestObservation } from './session-auto-healer.js';
 import type { ExecuteTurnFn } from './message-runtime-types.js';
 import type {
   AgentType,
@@ -255,6 +256,18 @@ export function createExecuteTurn(deps: CreateExecuteTurnDeps): ExecuteTurnFn {
       const { deliverySucceeded, visiblePhase } =
         await turnController.finish(outputStatus);
 
+      await notifyOwnerCodexBadRequestObservation({
+        chatJid,
+        runId,
+        groupFolder: group.folder,
+        channel,
+        outputStatus,
+        visiblePhase,
+        deliveryRole: resolvedDeliveryRole,
+        agentType: args.forcedAgentType ?? group.agentType ?? 'claude-code',
+        pairedTurnIdentity: args.pairedTurnIdentity ?? null,
+      });
+
       if (deliverySucceeded) {
         await deps.afterDeliverySuccess?.({
           chatJid,

src/message-runtime.test.ts

@@ -165,6 +165,7 @@ vi.mock('./db.js', () => {
     getLatestPreviousPairedTaskForChat: vi.fn(() => undefined),
     getPairedTaskById: vi.fn(() => undefined),
     getPairedTurnOutputs: vi.fn(() => []),
+    getOwnerCodexBadRequestFailureSummaryForTask: vi.fn(() => null),
     getRecentChatMessages: vi.fn(() => []),
     createProducedWorkItem: vi.fn((input) => ({
       id: 1,

src/session-auto-healer.test.ts

@@ -0,0 +1,166 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import {
+  _initTestDatabase,
+  createPairedTask,
+  failPairedTurn,
+  markPairedTurnRunning,
+} from './db.js';
+import { CODEX_MAIN_SERVICE_ID } from './config.js';
+import {
+  getCodexBadRequestRepeatThreshold,
+  notifyOwnerCodexBadRequestObservation,
+} from './session-auto-healer.js';
+import type { PairedTask } from './types.js';
+
+const BAD_REQUEST_ERROR = '{"detail":"Bad Request"}';
+const originalThresholdEnv = process.env.CODEX_BAD_REQUEST_REPEAT_THRESHOLD;
+
+const pairedTurnIdentity = {
+  turnId: 'task-1:updated:owner-follow-up',
+  taskId: 'task-1',
+  taskUpdatedAt: '2026-04-29T01:00:00.000Z',
+  intentKind: 'owner-follow-up' as const,
+  role: 'owner' as const,
+};
+
+function makeTask(overrides: Partial<PairedTask> = {}): PairedTask {
+  return {
+    id: 'task-1',
+    chat_jid: 'dc:room',
+    group_folder: 'eyejokerdb-9',
+    owner_service_id: CODEX_MAIN_SERVICE_ID,
+    reviewer_service_id: 'claude',
+    title: null,
+    source_ref: null,
+    plan_notes: null,
+    review_requested_at: null,
+    round_trip_count: 1,
+    status: 'active',
+    arbiter_verdict: null,
+    arbiter_requested_at: null,
+    completion_reason: null,
+    created_at: '2026-04-29T08:00:00.000Z',
+    updated_at: pairedTurnIdentity.taskUpdatedAt,
+    ...overrides,
+  };
+}
+
+function recordOwnerCodexFailure(runId: string): void {
+  markPairedTurnRunning({
+    turnIdentity: pairedTurnIdentity,
+    executorServiceId: CODEX_MAIN_SERVICE_ID,
+    executorAgentType: 'codex',
+    runId,
+  });
+  failPairedTurn({
+    turnIdentity: pairedTurnIdentity,
+    error: BAD_REQUEST_ERROR,
+  });
+}
+
+describe('session-auto-healer', () => {
+  beforeEach(() => {
+    _initTestDatabase();
+    delete process.env.CODEX_BAD_REQUEST_REPEAT_THRESHOLD;
+  });
+
+  afterEach(() => {
+    if (originalThresholdEnv === undefined) {
+      delete process.env.CODEX_BAD_REQUEST_REPEAT_THRESHOLD;
+    } else {
+      process.env.CODEX_BAD_REQUEST_REPEAT_THRESHOLD = originalThresholdEnv;
+    }
+  });
+
+  it('uses a conservative default Bad Request repeat threshold', () => {
+    expect(getCodexBadRequestRepeatThreshold()).toBe(3);
+  });
+
+  it('supports threshold override for observation tuning', () => {
+    process.env.CODEX_BAD_REQUEST_REPEAT_THRESHOLD = '2';
+
+    expect(getCodexBadRequestRepeatThreshold()).toBe(2);
+  });
+
+  it('ignores invalid threshold overrides', () => {
+    process.env.CODEX_BAD_REQUEST_REPEAT_THRESHOLD = '1';
+
+    expect(getCodexBadRequestRepeatThreshold()).toBe(3);
+  });
+
+  it('notifies once when owner Codex reaches the Bad Request observation threshold', async () => {
+    createPairedTask(makeTask());
+    for (const runId of ['run-1', 'run-2', 'run-3']) {
+      recordOwnerCodexFailure(runId);
+    }
+    const channel = { sendMessage: vi.fn() };
+
+    const notified = await notifyOwnerCodexBadRequestObservation({
+      chatJid: 'dc:room',
+      runId: 'run-1',
+      groupFolder: 'eyejokerdb-9',
+      channel,
+      outputStatus: 'error',
+      visiblePhase: 'silent',
+      deliveryRole: 'owner',
+      agentType: 'codex',
+      pairedTurnIdentity,
+      threshold: 3,
+    });
+
+    expect(notified).toBe(true);
+    expect(channel.sendMessage).toHaveBeenCalledWith(
+      'dc:room',
+      expect.stringContaining('자동복구는 아직 비활성화'),
+    );
+    expect(channel.sendMessage).toHaveBeenCalledWith(
+      'dc:room',
+      expect.stringContaining('task=task-1'),
+    );
+  });
+
+  it('does not notify when the failure had visible output', async () => {
+    const channel = { sendMessage: vi.fn() };
+
+    const notified = await notifyOwnerCodexBadRequestObservation({
+      chatJid: 'dc:room',
+      runId: 'run-1',
+      groupFolder: 'eyejokerdb-9',
+      channel,
+      outputStatus: 'error',
+      visiblePhase: 'progress',
+      deliveryRole: 'owner',
+      agentType: 'codex',
+      pairedTurnIdentity,
+      threshold: 3,
+    });
+
+    expect(notified).toBe(false);
+    expect(channel.sendMessage).not.toHaveBeenCalled();
+  });
+
+  it('does not notify again after the exact threshold has already passed', async () => {
+    createPairedTask(makeTask());
+    for (const runId of ['run-1', 'run-2', 'run-3', 'run-4']) {
+      recordOwnerCodexFailure(runId);
+    }
+    const channel = { sendMessage: vi.fn() };
+
+    const notified = await notifyOwnerCodexBadRequestObservation({
+      chatJid: 'dc:room',
+      runId: 'run-1',
+      groupFolder: 'eyejokerdb-9',
+      channel,
+      outputStatus: 'error',
+      visiblePhase: 'silent',
+      deliveryRole: 'owner',
+      agentType: 'codex',
+      pairedTurnIdentity,
+      threshold: 3,
+    });
+
+    expect(notified).toBe(false);
+    expect(channel.sendMessage).not.toHaveBeenCalled();
+  });
+});

src/session-auto-healer.ts

@@ -0,0 +1,97 @@
+import { getOwnerCodexBadRequestFailureSummaryForTask } from './db.js';
+import { getEnv } from './env.js';
+import { logger } from './logger.js';
+import type { PairedTurnIdentity } from './paired-turn-identity.js';
+import type {
+  AgentType,
+  Channel,
+  PairedRoomRole,
+  VisiblePhase,
+} from './types.js';
+
+const DEFAULT_CODEX_BAD_REQUEST_REPEAT_THRESHOLD = 3;
+
+export function getCodexBadRequestRepeatThreshold(): number {
+  const raw = getEnv('CODEX_BAD_REQUEST_REPEAT_THRESHOLD');
+  if (!raw) {
+    return DEFAULT_CODEX_BAD_REQUEST_REPEAT_THRESHOLD;
+  }
+
+  const parsed = Number.parseInt(raw, 10);
+  if (!Number.isFinite(parsed) || parsed < 2) {
+    return DEFAULT_CODEX_BAD_REQUEST_REPEAT_THRESHOLD;
+  }
+  return parsed;
+}
+
+export async function notifyOwnerCodexBadRequestObservation(args: {
+  chatJid: string;
+  runId: string;
+  groupFolder: string;
+  channel: Pick<Channel, 'sendMessage'>;
+  outputStatus: 'success' | 'error';
+  visiblePhase: VisiblePhase;
+  deliveryRole: PairedRoomRole | null;
+  agentType: AgentType;
+  pairedTurnIdentity: PairedTurnIdentity | null;
+  threshold?: number;
+}): Promise<boolean> {
+  if (
+    args.outputStatus !== 'error' ||
+    args.visiblePhase !== 'silent' ||
+    args.agentType !== 'codex'
+  ) {
+    return false;
+  }
+
+  const role = args.pairedTurnIdentity?.role ?? args.deliveryRole;
+  if (role !== 'owner' || !args.pairedTurnIdentity) {
+    return false;
+  }
+
+  const threshold = args.threshold ?? getCodexBadRequestRepeatThreshold();
+  const summary = getOwnerCodexBadRequestFailureSummaryForTask({
+    taskId: args.pairedTurnIdentity.taskId,
+    threshold,
+  });
+  if (!summary || summary.failures !== threshold) {
+    return false;
+  }
+
+  const message = [
+    `🟡 owner Codex 세션이 {"detail":"Bad Request"}로 ${summary.failures}회 연속 무출력 실패했습니다.`,
+    '자동복구는 아직 비활성화되어 있습니다. 현재는 감지만 수행하며, 필요하면 `/clear`로 세션을 초기화해 주세요.',
+    `task=${summary.taskId}`,
+    `latest=${summary.latestFailureAt}`,
+  ].join('\n');
+
+  try {
+    await args.channel.sendMessage(args.chatJid, message);
+  } catch (err) {
+    logger.warn(
+      {
+        err,
+        chatJid: args.chatJid,
+        runId: args.runId,
+        groupFolder: args.groupFolder,
+        taskId: summary.taskId,
+      },
+      'Failed to send owner Codex Bad Request observation notice',
+    );
+    return false;
+  }
+
+  logger.warn(
+    {
+      chatJid: args.chatJid,
+      runId: args.runId,
+      groupFolder: args.groupFolder,
+      taskId: summary.taskId,
+      failures: summary.failures,
+      firstFailureAt: summary.firstFailureAt,
+      latestFailureAt: summary.latestFailureAt,
+    },
+    'Detected repeated owner Codex Bad Request failures without visible output',
+  );
+  return true;
+}

src/session-recovery.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, it } from 'vitest';
 
 import {
+  isCodexBadRequestSignal,
   shouldResetCodexSessionOnAgentFailure,
   shouldResetSessionOnAgentFailure,
   shouldRetryFreshCodexSessionOnAgentFailure,
@@ -133,4 +134,24 @@ describe('shouldRetryFreshCodexSessionOnAgentFailure', () => {
       }),
     ).toBe(true);
   });
+
+  it('does not retry generic Codex Bad Request signals during observation-only rollout', () => {
+    const output = {
+      result: null,
+      error: '{"detail":"Bad Request"}',
+    };
+
+    expect(isCodexBadRequestSignal(output)).toBe(true);
+    expect(shouldResetCodexSessionOnAgentFailure(output)).toBe(false);
+    expect(shouldRetryFreshCodexSessionOnAgentFailure(output)).toBe(false);
+  });
+
+  it('does not flag unrelated Bad Request text as the narrow Codex signal', () => {
+    expect(
+      isCodexBadRequestSignal({
+        result: null,
+        error: 'HTTP 400 Bad Request',
+      }),
+    ).toBe(false);
+  });
 });

src/session-recovery.ts

@@ -1,5 +1,6 @@
 import { getAgentOutputText } from './agent-output.js';
 import type { AgentOutput } from './agent-runner.js';
+import { isCodexBadRequestText } from './codex-bad-request-signal.js';
 
 const SESSION_RESET_PATTERNS = [
   /An image in the conversation exceeds the dimension limit for many-image requests \(2000px\)\./i,
@@ -74,3 +75,13 @@ export function shouldRetryFreshCodexSessionOnAgentFailure(
 ): boolean {
   return shouldResetCodexSessionOnAgentFailure(output);
 }
+
+export function isCodexBadRequestSignal(
+  output: Pick<AgentOutput, 'result' | 'output' | 'error'>,
+): boolean {
+  const texts = [
+    ...toText(getAgentOutputText(output)),
+    ...toText(output.error),
+  ];
+  return texts.some((text) => isCodexBadRequestText(text));
+}