merge: integrate origin/main (406 commits) into live branch

Merge upstream's paired-room stabilization, dedicated message-runtime-loop,
outbound delivery refactor (ipc-outbound-delivery / deliverFormattedCanonicalMessage),
discord module split, dashboard rework, and migrations 015-018 into our branch.

Conflict policy: prefer upstream for the heavily-refactored paired-room /
message-runtime / db cluster (it supersedes our earlier loop band-aids), keep
only orthogonal unique fixes:
- codex bundled-JS launcher fix (codex-warmup.ts)
- slot-aligned Claude+Codex usage primer (usage-primer.ts) re-wired into index.ts
- MemoryHigh=3G cgroup bound, discord perms diagnostic, prompt docs
- progress null-race + silent-failure publish guards (message-turn-controller.ts)

Dropped (superseded by upstream or unwired): reviewer STEP_DONE/TASK_DONE loop
band-aids, router markdown-escape override, register tribunal-default+git-init,
restart-context rewindToSeq (unwired).

Verified: typecheck clean, build clean, vitest shows zero new regressions vs the
origin/main baseline (only pre-existing env-config failures remain).

src/index.ts

@@ -1,10 +1,12 @@
 import {
   ASSISTANT_NAME,
-  DATA_DIR,
+  ARBITER_SERVICE_ID,
+  CLAUDE_SERVICE_ID,
+  CODEX_MAIN_SERVICE_ID,
+  CODEX_REVIEW_SERVICE_ID,
   IDLE_TIMEOUT,
   POLL_INTERVAL,
   SERVICE_ID,
-  isSessionCommandSenderAllowed,
   STATUS_CHANNEL_ID,
   STATUS_UPDATE_INTERVAL,
   TIMEZONE,
@@ -19,29 +21,25 @@ import {
 } from './channels/registry.js';
 import { writeGroupsSnapshot } from './agent-runner.js';
 import {
-  type AssignRoomInput,
-  getAllTasks,
-  getEarliestUnansweredHumanSeq,
   hasRecentRestartAnnouncement,
   initDatabase,
+  recoverInterruptedPairedTurnAttemptsForService,
   storeChatMetadata,
   storeMessage,
 } from './db.js';
-import { composeDashboardContent } from './dashboard-render.js';
 import { GroupQueue } from './group-queue.js';
 import { resolveGroupIpcPath } from './group-folder.js';
 import { startIpcWatcher } from './ipc.js';
 import {
-  findChannel,
-  findChannelByName,
-  sanitizeForOutbound,
-  normalizeMessageForDedupe,
-  resolveChannelForDeliveryRole,
-} from './router.js';
+  deliverCanonicalOutboundMessage,
+  deliverIpcOutboundMessage,
+} from './ipc-outbound-delivery.js';
+import { findChannel, formatOutbound } from './router.js';
 import {
   buildRestartAnnouncement,
   buildInterruptedRestartAnnouncement,
   consumeRestartContext,
+  getRecoverableInterruptedGroups,
   getInterruptedRecoveryCandidates,
   inferRecentRestartContext,
   type RestartContext,
@@ -49,7 +47,6 @@ import {
 } from './restart-context.js';
 import {
   isSenderAllowed,
-  isTriggerAllowed,
   loadSenderAllowlist,
   shouldDropMessage,
 } from './sender-allowlist.js';
@@ -61,6 +58,10 @@ import { startWebDashboardServer } from './web-dashboard-server.js';
 import { Channel, NewMessage, RegisteredGroup } from './types.js';
 import { logger } from './logger.js';
 import { initCodexTokenRotation } from './codex-token-rotation.js';
+import {
+  startCodexAccountRefreshLoop,
+  stopCodexAccountRefreshLoop,
+} from './settings-store.js';
 import {
   hasAvailableClaudeToken,
   initTokenRotation,
@@ -69,7 +70,7 @@ import {
   isBotMessageSourceKind,
   resolveInjectedMessageSourceKind,
 } from './message-source.js';
-import { parseVisibleVerdict } from './paired-execution-context-shared.js';
+import { parseVisibleVerdict } from './paired-verdict.js';
 
 export function isTerminalStatusMessage(text: string): boolean {
   return parseVisibleVerdict(text) !== 'continue';
@@ -88,20 +89,6 @@ import { FAILOVER_MIN_DURATION_MS } from './config.js';
 
 // Token rotation is initialized lazily on first use or at startup below
 
-export async function sendFormattedChannelMessage(
-  channels: Channel[],
-  jid: string,
-  rawText: string,
-): Promise<void> {
-  const channel = findChannel(channels, jid);
-  if (!channel) {
-    logger.warn({ jid }, 'No channel owns JID, cannot send message');
-    return;
-  }
-  const text = sanitizeForOutbound(rawText);
-  if (text) await channel.sendMessage(jid, text);
-}
-
 export async function sendFormattedTrackedChannelMessage(
   channels: Channel[],
   jid: string,
@@ -112,7 +99,7 @@ export async function sendFormattedTrackedChannelMessage(
     logger.warn({ jid }, 'No channel owns JID, cannot send tracked message');
     return null;
   }
-  const text = sanitizeForOutbound(rawText);
+  const text = formatOutbound(rawText);
   if (!text || !channel.sendAndTrack) return null;
   return channel.sendAndTrack(jid, text);
 }
@@ -128,7 +115,7 @@ export async function editFormattedTrackedChannelMessage(
     logger.warn({ jid }, 'No channel owns JID, cannot edit tracked message');
     return;
   }
-  const text = sanitizeForOutbound(rawText);
+  const text = formatOutbound(rawText);
   if (!text || !channel.editMessage) return;
   await channel.editMessage(jid, messageId, text);
 }
@@ -154,6 +141,23 @@ const runtime = createMessageRuntime({
   clearSession: runtimeState.clearSession,
 });
 
+async function deliverFormattedCanonicalMessage(
+  jid: string,
+  rawText: string,
+  deliveryRole?: 'owner' | 'reviewer' | 'arbiter',
+): Promise<void> {
+  const text = formatOutbound(rawText);
+  if (!text) return;
+  await deliverCanonicalOutboundMessage(
+    { jid, text, deliveryRole },
+    {
+      channels,
+      roomBindings: runtimeState.getRoomBindings,
+      log: logger,
+    },
+  );
+}
+
 /**
  * Get available groups list for the agent.
  * Returns groups ordered by most recent activity.
@@ -176,6 +180,22 @@ export function _setRoomBindings(
   runtimeState.setRoomBindings(groups);
 }
 
+async function tryDeliverRestartAnnouncement(
+  chatJid: string,
+  rawText: string,
+): Promise<boolean> {
+  try {
+    await deliverFormattedCanonicalMessage(chatJid, rawText);
+    return true;
+  } catch (err) {
+    logger.warn(
+      { err, chatJid },
+      'Skipped restart recovery announcement because no channel is connected',
+    );
+    return false;
+  }
+}
+
 async function announceRestartRecovery(
   processStartedAtMs: number,
 ): Promise<RestartContext | null> {
@@ -190,23 +210,26 @@ async function announceRestartRecovery(
       return explicitContext;
     }
 
-    await sendFormattedChannelMessage(
-      channels,
-      explicitContext.chatJid,
-      buildRestartAnnouncement(explicitContext),
-    );
-    logger.info(
-      { chatJid: explicitContext.chatJid },
-      'Sent explicit restart recovery announcement',
-    );
+    if (
+      await tryDeliverRestartAnnouncement(
+        explicitContext.chatJid,
+        buildRestartAnnouncement(explicitContext),
+      )
+    ) {
+      logger.info(
+        { chatJid: explicitContext.chatJid },
+        'Sent explicit restart recovery announcement',
+      );
+    }
 
-    for (const interrupted of explicitContext.interruptedGroups ?? []) {
+    for (const interrupted of getRecoverableInterruptedGroups(
+      explicitContext,
+    )) {
       if (interrupted.chatJid === explicitContext.chatJid) continue;
       if (hasRecentRestartAnnouncement(interrupted.chatJid, dedupeSince)) {
         continue;
       }
-      await sendFormattedChannelMessage(
-        channels,
+      await tryDeliverRestartAnnouncement(
         interrupted.chatJid,
         buildInterruptedRestartAnnouncement(interrupted),
       );
@@ -228,18 +251,65 @@ async function announceRestartRecovery(
     return null;
   }
 
-  await sendFormattedChannelMessage(
-    channels,
-    inferred.chatJid,
-    inferred.lines.join('\n'),
-  );
-  logger.info(
-    { chatJid: inferred.chatJid },
-    'Sent inferred restart recovery announcement',
-  );
+  if (
+    await tryDeliverRestartAnnouncement(
+      inferred.chatJid,
+      inferred.lines.join('\n'),
+    )
+  ) {
+    logger.info(
+      { chatJid: inferred.chatJid },
+      'Sent inferred restart recovery announcement',
+    );
+  }
   return null;
 }
 
+function queueInterruptedPairedTurnAttemptRecovery(): void {
+  for (const candidate of recoverInterruptedPairedTurnAttemptsForService({
+    serviceIds: [
+      SERVICE_ID,
+      CLAUDE_SERVICE_ID,
+      CODEX_MAIN_SERVICE_ID,
+      CODEX_REVIEW_SERVICE_ID,
+      ARBITER_SERVICE_ID,
+    ].filter((serviceId): serviceId is string => Boolean(serviceId)),
+  })) {
+    const binding = runtimeState.getRoomBindings()[candidate.chat_jid];
+    if (!binding) {
+      logger.warn(
+        {
+          chatJid: candidate.chat_jid,
+          groupFolder: candidate.group_folder,
+          taskId: candidate.task_id,
+          turnId: candidate.turn_id,
+          attemptId: candidate.attempt_id,
+        },
+        'Skipped interrupted paired turn recovery because room binding is missing',
+      );
+      continue;
+    }
+    queue.enqueueMessageCheck(
+      candidate.chat_jid,
+      resolveGroupIpcPath(binding.folder),
+    );
+    logger.info(
+      {
+        chatJid: candidate.chat_jid,
+        groupFolder: binding.folder,
+        taskId: candidate.task_id,
+        taskStatus: candidate.task_status,
+        turnId: candidate.turn_id,
+        attemptId: candidate.attempt_id,
+        attemptNo: candidate.attempt_no,
+        role: candidate.role,
+        intentKind: candidate.intent_kind,
+      },
+      'Queued interrupted paired turn attempt for restart recovery',
+    );
+  }
+}
+
 async function main(): Promise<void> {
   const processStartedAtMs = Date.now();
   initDatabase();
@@ -247,6 +317,7 @@ async function main(): Promise<void> {
   initTokenRotation();
   initCodexTokenRotation();
   startTokenRefreshLoop();
+  startCodexAccountRefreshLoop();
 
   runtimeState.loadState();
 
@@ -256,6 +327,7 @@ async function main(): Promise<void> {
   const shutdown = async (signal: string) => {
     logger.info({ signal }, 'Shutdown signal received');
     stopTokenRefreshLoop();
+    stopCodexAccountRefreshLoop();
     if (leaseRecoveryTimer) {
       clearInterval(leaseRecoveryTimer);
       leaseRecoveryTimer = null;
@@ -269,32 +341,17 @@ async function main(): Promise<void> {
         (
           status,
         ): status is typeof status & {
-          status: 'processing' | 'waiting';
-        } => status.status !== 'inactive',
+          status: 'processing';
+        } => status.status === 'processing',
       )
-      .map((status) => {
-        // Capture the seq of the earliest unanswered human message for this
-        // chat so the next startup can rewind the agent cursor and re-run
-        // the interrupted turn automatically.
-        let rewindToSeq: number | null = null;
-        try {
-          rewindToSeq = getEarliestUnansweredHumanSeq(status.jid);
-        } catch (err) {
-          logger.warn(
-            { err, chatJid: status.jid },
-            'Failed to capture rewind seq for shutdown snapshot',
-          );
-        }
-        return {
-          chatJid: status.jid,
-          groupName: roomBindings[status.jid]?.name || status.jid,
-          status: status.status,
-          elapsedMs: status.elapsedMs,
-          pendingMessages: status.pendingMessages,
-          pendingTasks: status.pendingTasks,
-          rewindToSeq,
-        };
-      });
+      .map((status) => ({
+        chatJid: status.jid,
+        groupName: roomBindings[status.jid]?.name || status.jid,
+        status: status.status,
+        elapsedMs: status.elapsedMs,
+        pendingMessages: status.pendingMessages,
+        pendingTasks: status.pendingTasks,
+      }));
     const writtenPaths = writeShutdownRestartContext(
       roomBindings,
       interruptedGroups,
@@ -362,22 +419,28 @@ async function main(): Promise<void> {
       );
       continue;
     }
-    channels.push(channel);
-    await channel.connect();
+    try {
+      await channel.connect();
+      channels.push(channel);
+    } catch (err) {
+      logger.error(
+        { channel: channelName, err },
+        'Channel connect failed — skipping',
+      );
+    }
   }
   if (channels.length === 0) {
-    logger.fatal('No channels connected');
-    process.exit(1);
+    if (WEB_DASHBOARD.enabled) {
+      logger.warn(
+        'No channels connected; continuing in web-dashboard-only mode',
+      );
+    } else {
+      logger.fatal('No channels connected');
+      process.exit(1);
+    }
   }
 
   // Start subsystems (independently of connection handler)
-  // Paired-room scheduler output goes through the reviewer bot slot.
-  const reviewerChannelName = 'discord-review';
-  const reviewerChannelForCron = findChannelByName(
-    channels,
-    reviewerChannelName,
-  );
-
   startSchedulerLoop({
     roomBindings: runtimeState.getRoomBindings,
     getSessions: runtimeState.getSessions,
@@ -385,58 +448,25 @@ async function main(): Promise<void> {
     onProcess: (groupJid, proc, processName, ipcDir) =>
       queue.registerProcess(groupJid, proc, processName, ipcDir),
     sendMessage: (jid, rawText) =>
-      sendFormattedChannelMessage(channels, jid, rawText),
-    sendMessageViaReviewerBot: reviewerChannelForCron
-      ? async (jid, rawText) => {
-          const text = sanitizeForOutbound(rawText);
-          if (text) await reviewerChannelForCron.sendMessage(jid, text);
-        }
-      : undefined,
+      deliverFormattedCanonicalMessage(jid, rawText),
+    sendMessageViaReviewerBot: (jid, rawText) =>
+      deliverFormattedCanonicalMessage(jid, rawText, 'reviewer'),
     sendTrackedMessage: (jid, rawText) =>
       sendFormattedTrackedChannelMessage(channels, jid, rawText),
     editTrackedMessage: (jid, messageId, rawText) =>
       editFormattedTrackedChannelMessage(channels, jid, messageId, rawText),
   });
   startIpcWatcher({
-    sendMessage: async (jid, text, senderRole, runId) => {
-      if (
-        runId &&
-        (senderRole === 'reviewer' || senderRole === 'arbiter') &&
-        queue.hasRecordedDirectTerminalDeliveryForRun(jid, runId, senderRole)
-      ) {
-        logger.info(
-          {
-            transition: 'ipc:skip-post-terminal',
-            chatJid: jid,
-            senderRole,
-            runId,
-          },
-          'Skipped IPC relay message because the run already emitted a direct terminal verdict',
-        );
-        return;
-      }
-      const route = resolveChannelForDeliveryRole(channels, jid, senderRole);
-      if (!route.channel) throw new Error(`No channel for JID: ${jid}`);
-      logger.info(
+    sendMessage: async (jid, text, senderRole, runId, attachments) => {
+      await deliverIpcOutboundMessage(
+        { jid, text, senderRole, runId, attachments },
         {
-          transition: 'ipc:route',
-          chatJid: jid,
-          runId: runId ?? null,
-          senderRole: senderRole ?? null,
-          requestedRoleChannel: route.requestedRoleChannelName,
-          selectedChannel: route.selectedChannelName,
-          usedRoleChannel: route.usedRoleChannel,
-          fallbackUsed: route.fallbackUsed,
+          channels,
+          roomBindings: runtimeState.getRoomBindings,
+          queue,
+          log: logger,
         },
-        'IPC relay routed message to channel',
       );
-      await route.channel.sendMessage(jid, text);
-      if (
-        (senderRole === 'reviewer' || senderRole === 'arbiter') &&
-        isTerminalStatusMessage(text)
-      ) {
-        queue.noteDirectTerminalDelivery(jid, senderRole, text);
-      }
     },
     injectInboundMessage: async (payload) => {
       const jid = payload.chatJid;
@@ -498,6 +528,7 @@ async function main(): Promise<void> {
   });
   queue.setProcessMessagesFn(runtime.processGroupMessages);
   queue.enterRecoveryMode();
+  queueInterruptedPairedTurnAttemptRecovery();
   runtime.recoverPendingMessages();
   const restartContext = await announceRestartRecovery(processStartedAtMs);
   const roomBindings = runtimeState.getRoomBindings();
@@ -505,37 +536,6 @@ async function main(): Promise<void> {
     restartContext,
     roomBindings,
   )) {
-    // Auto-resume: rewind the agent cursor to just before the earliest
-    // unanswered human message so the missed-message processor picks it up
-    // again. Claude/Codex sessions are persisted by group folder, so the
-    // agent will resume mid-turn rather than restarting from scratch.
-    if (candidate.rewindToSeq != null && candidate.rewindToSeq > 0) {
-      const cursors = runtimeState.getLastAgentTimestamps();
-      const previousCursor = cursors[candidate.chatJid];
-      const targetCursor = String(candidate.rewindToSeq - 1);
-      // Only rewind if the current cursor is past the target — never advance.
-      const previousNum = previousCursor
-        ? Number.parseInt(previousCursor, 10)
-        : 0;
-      if (
-        Number.isFinite(previousNum) &&
-        previousNum >= candidate.rewindToSeq
-      ) {
-        cursors[candidate.chatJid] = targetCursor;
-        runtimeState.saveState();
-        logger.info(
-          {
-            chatJid: candidate.chatJid,
-            groupFolder: candidate.groupFolder,
-            previousCursor,
-            rewoundTo: targetCursor,
-            rewindToSeq: candidate.rewindToSeq,
-          },
-          'Rewound agent cursor for interrupted-turn auto-resume',
-        );
-      }
-    }
-
     queue.enqueueMessageCheck(
       candidate.chatJid,
       resolveGroupIpcPath(candidate.groupFolder),
@@ -547,7 +547,6 @@ async function main(): Promise<void> {
         status: candidate.status,
         pendingMessages: candidate.pendingMessages,
         pendingTasks: candidate.pendingTasks,
-        rewindToSeq: candidate.rewindToSeq,
       },
       'Queued interrupted group for restart recovery',
     );
@@ -570,7 +569,13 @@ async function main(): Promise<void> {
     },
     purgeOnStart: true,
   });
-  webDashboardServer = startWebDashboardServer(WEB_DASHBOARD);
+  webDashboardServer = startWebDashboardServer({
+    ...WEB_DASHBOARD,
+    getRoomBindings: runtimeState.getRoomBindings,
+    enqueueMessageCheck: (chatJid, groupFolder) =>
+      queue.enqueueMessageCheck(chatJid, resolveGroupIpcPath(groupFolder)),
+    nudgeScheduler: nudgeSchedulerLoop,
+  });
   startUsagePrimer();
 
   leaseRecoveryTimer = setInterval(() => {