From 778ed9b94a29b97c0ec735a2f3349b4a17195484 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Sun, 31 May 2026 17:17:38 +0900 Subject: [PATCH] fix: preserve paired input evidence context (#204) --- runners/agent-runner/src/output-protocol.ts | 6 +- .../agent-runner/test/output-protocol.test.ts | 53 ++++++++++++ runners/codex-runner/src/app-server-input.ts | 6 +- .../test/app-server-input.test.ts | 32 +++++++ runners/shared/src/agent-protocol.ts | 30 +++++++ runners/shared/src/index.ts | 1 + runners/shared/test/agent-protocol.test.ts | 18 ++++ src/message-runtime-flow.ts | 20 ++--- src/message-runtime-prompts.ts | 2 +- src/message-runtime-queue.ts | 22 +++-- src/message-runtime-task-context.test.ts | 85 +++++++++++++++++++ src/message-runtime-task-context.ts | 35 ++++++++ 12 files changed, 287 insertions(+), 23 deletions(-) create mode 100644 src/message-runtime-task-context.test.ts create mode 100644 src/message-runtime-task-context.ts diff --git a/runners/agent-runner/src/output-protocol.ts b/runners/agent-runner/src/output-protocol.ts index f96819e..a64d51a 100644 --- a/runners/agent-runner/src/output-protocol.ts +++ b/runners/agent-runner/src/output-protocol.ts @@ -2,6 +2,7 @@ import fs from 'fs'; import path from 'path'; import { + expandImagePromptReferences, extractImageTagPaths, imageTagCaption, missingImageTagCaption, @@ -79,7 +80,8 @@ export function buildMultimodalContent( text: string, log: LogFn, ): StreamContent { - const { imagePaths } = extractImageTagPaths(text); + const expandedText = expandImagePromptReferences(text); + const { imagePaths } = extractImageTagPaths(expandedText); if (imagePaths.length === 0) return text; const blocks: ContentBlock[] = []; @@ -88,7 +90,7 @@ export function buildMultimodalContent( if (trimmed) blocks.push({ type: 'text', text: trimmed }); }; - for (const part of splitImageTagPromptParts(text)) { + for (const part of splitImageTagPromptParts(expandedText)) { if (part.type === 'text') { pushText(part.text); continue; diff --git a/runners/agent-runner/test/output-protocol.test.ts b/runners/agent-runner/test/output-protocol.test.ts index aa33582..44c8c52 100644 --- a/runners/agent-runner/test/output-protocol.test.ts +++ b/runners/agent-runner/test/output-protocol.test.ts @@ -94,4 +94,57 @@ describe('agent-runner multimodal prompts', () => { ]); expect(logs).toContain(`Unsupported image type, skipping: ${imagePath}`); }); + + it('loads MEDIA image directives as Claude image blocks', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ejclaw-media-image-')); + cleanupDirs.push(dir); + const imagePath = path.join(dir, 'media-render.png'); + fs.writeFileSync(imagePath, ONE_PIXEL_PNG); + + const content = buildMultimodalContent(`증거\nMEDIA:${imagePath}`, () => { + // no-op + }); + + expect(content).toEqual([ + { type: 'text', text: '증거' }, + { type: 'text', text: 'Image evidence: media-render.png' }, + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: ONE_PIXEL_PNG.toString('base64'), + }, + }, + ]); + }); + + it('loads markdown image links as Claude image blocks', () => { + const dir = fs.mkdtempSync( + path.join(os.tmpdir(), 'ejclaw-markdown-image-'), + ); + cleanupDirs.push(dir); + const imagePath = path.join(dir, 'markdown-render.png'); + fs.writeFileSync(imagePath, ONE_PIXEL_PNG); + + const content = buildMultimodalContent( + `증거 ![render](${imagePath})`, + () => { + // no-op + }, + ); + + expect(content).toEqual([ + { type: 'text', text: '증거' }, + { type: 'text', text: 'Image evidence: markdown-render.png' }, + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: ONE_PIXEL_PNG.toString('base64'), + }, + }, + ]); + }); }); diff --git a/runners/codex-runner/src/app-server-input.ts b/runners/codex-runner/src/app-server-input.ts index aee6eb2..6651785 100644 --- a/runners/codex-runner/src/app-server-input.ts +++ b/runners/codex-runner/src/app-server-input.ts @@ -2,6 +2,7 @@ import fs from 'fs'; import path from 'path'; import { + expandImagePromptReferences, extractImageTagPaths, imageTagCaption, missingImageTagCaption, @@ -22,7 +23,8 @@ export function parseAppServerInput( text: string, log: (message: string) => void = () => undefined, ): AppServerInputItem[] { - const { imagePaths } = extractImageTagPaths(text); + const expandedText = expandImagePromptReferences(text); + const { imagePaths } = extractImageTagPaths(expandedText); const input: AppServerInputItem[] = []; const pushText = (value: string) => { const trimmed = value.trim(); @@ -30,7 +32,7 @@ export function parseAppServerInput( }; if (imagePaths.length > 0) { - for (const part of splitImageTagPromptParts(text)) { + for (const part of splitImageTagPromptParts(expandedText)) { if (part.type === 'text') { pushText(part.text); continue; diff --git a/runners/codex-runner/test/app-server-input.test.ts b/runners/codex-runner/test/app-server-input.test.ts index b5434d7..272c1cc 100644 --- a/runners/codex-runner/test/app-server-input.test.ts +++ b/runners/codex-runner/test/app-server-input.test.ts @@ -86,4 +86,36 @@ describe('codex app-server input', () => { ]); expect(logs).toContain(`Unsupported image type, skipping: ${imagePath}`); }); + + it('loads MEDIA image directives as local image input items', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ejclaw-codex-media-')); + cleanupDirs.push(dir); + const imagePath = path.join(dir, 'media-render.png'); + fs.writeFileSync(imagePath, ONE_PIXEL_PNG); + + const input = parseAppServerInput(`증거\nMEDIA:${imagePath}`); + + expect(input).toEqual([ + { type: 'text', text: '증거' }, + { type: 'text', text: 'Image evidence: media-render.png' }, + { type: 'localImage', path: imagePath }, + ]); + }); + + it('loads markdown image links as local image input items', () => { + const dir = fs.mkdtempSync( + path.join(os.tmpdir(), 'ejclaw-codex-markdown-'), + ); + cleanupDirs.push(dir); + const imagePath = path.join(dir, 'markdown-render.png'); + fs.writeFileSync(imagePath, ONE_PIXEL_PNG); + + const input = parseAppServerInput(`증거 ![render](${imagePath})`); + + expect(input).toEqual([ + { type: 'text', text: '증거' }, + { type: 'text', text: 'Image evidence: markdown-render.png' }, + { type: 'localImage', path: imagePath }, + ]); + }); }); diff --git a/runners/shared/src/agent-protocol.ts b/runners/shared/src/agent-protocol.ts index 44d7c00..dccf798 100644 --- a/runners/shared/src/agent-protocol.ts +++ b/runners/shared/src/agent-protocol.ts @@ -92,6 +92,36 @@ export function extractImageTagPaths(text: string): { }; } +export function expandImagePromptReferences(text: string): string { + const codeSpans = fencedCodeSpans(text); + const withMediaImages = text.replace( + MEDIA_TAG_RE, + (full: string, doubleQuoted, singleQuoted, backticked, bare, offset) => { + if (isInsideSpans(offset, codeSpans)) return full; + const filePath = String( + doubleQuoted ?? singleQuoted ?? backticked ?? bare ?? '', + ).trim(); + if (!filePath.startsWith('/') || !IMAGE_EXT_RE.test(filePath)) { + return full; + } + const name = attachmentName(filePath) ?? filePath; + return `[Image: ${name} → ${filePath}]`; + }, + ); + + const markdownCodeSpans = fencedCodeSpans(withMediaImages); + return withMediaImages.replace( + MARKDOWN_IMAGE_ABSOLUTE_LINK_RE, + (full: string, rawPath: string, offset: number) => { + if (isInsideSpans(offset, markdownCodeSpans)) return full; + const filePath = rawPath.trim(); + if (!IMAGE_EXT_RE.test(filePath)) return full; + const name = attachmentName(filePath) ?? filePath; + return `[Image: ${name} → ${filePath}]`; + }, + ); +} + export type ImageTagPromptPart = | { type: 'text'; text: string } | { type: 'image'; label: string | null; path: string; raw: string }; diff --git a/runners/shared/src/index.ts b/runners/shared/src/index.ts index d5ac68a..cdcde68 100644 --- a/runners/shared/src/index.ts +++ b/runners/shared/src/index.ts @@ -38,6 +38,7 @@ export { type TaskContextMode, } from './task-runtime.js'; export { + expandImagePromptReferences, extractMarkdownImageAttachments, extractMediaAttachments, extractImageTagPaths, diff --git a/runners/shared/test/agent-protocol.test.ts b/runners/shared/test/agent-protocol.test.ts index dd0f752..97a689f 100644 --- a/runners/shared/test/agent-protocol.test.ts +++ b/runners/shared/test/agent-protocol.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import { + expandImagePromptReferences, extractImageTagPaths, imageTagCaption, missingImageTagCaption, @@ -59,6 +60,23 @@ describe('shared agent protocol helpers', () => { ); }); + it('expands MEDIA image directives into image prompt tags', () => { + expect(expandImagePromptReferences('증거\nMEDIA:/tmp/render.png\n끝')).toBe( + '증거\n[Image: render.png → /tmp/render.png]\n끝', + ); + }); + + it('expands markdown image links into image prompt tags', () => { + expect( + expandImagePromptReferences('증거 ![render](/tmp/render.png) 끝'), + ).toBe('증거 [Image: render.png → /tmp/render.png] 끝'); + }); + + it('keeps non-image media and fenced media text unchanged', () => { + const text = 'MEDIA:/tmp/demo.mp4\n```text\nMEDIA:/tmp/render.png\n```'; + expect(expandImagePromptReferences(text)).toBe(text); + }); + it('normalizes plain text runner output as public text', () => { expect(normalizePublicTextOutput('DONE')).toEqual({ result: 'DONE', diff --git a/src/message-runtime-flow.ts b/src/message-runtime-flow.ts index e7d937c..6a4d9b3 100644 --- a/src/message-runtime-flow.ts +++ b/src/message-runtime-flow.ts @@ -1,8 +1,4 @@ -import { - getLastHumanMessageContent, - getPairedTurnOutputs, - getRecentChatMessages, -} from './db.js'; +import { getLastHumanMessageContent, getPairedTurnOutputs } from './db.js'; import { logger } from './logger.js'; import { buildArbiterPromptForTask, @@ -22,6 +18,7 @@ import { resolveNextTurnAction, } from './message-runtime-rules.js'; import type { ExecuteTurnFn } from './message-runtime-types.js'; +import { getTaskContextMessages } from './message-runtime-task-context.js'; import { buildPairedTurnIdentity } from './paired-turn-identity.js'; import { resolveStoredVisibleVerdict } from './paired-verdict.js'; import { @@ -106,7 +103,6 @@ export function buildPendingPairedTurn(args: { timezone, task, rawMissedMessages, - recentHumanMessages, labeledRecentMessages, resolveChannel, } = args; @@ -124,7 +120,7 @@ export function buildPendingPairedTurn(args: { }), ownerFailureCount: task.owner_failure_count, }); - const recentMessages = getRecentChatMessages(chatJid, 20); + const taskContextMessages = getTaskContextMessages(chatJid, task); const lastHumanMessage = getLastHumanMessageContent(chatJid); if (nextTurnAction.kind === 'reviewer-turn') { @@ -133,7 +129,9 @@ export function buildPendingPairedTurn(args: { chatJid, timezone, turnOutputs, - recentHumanMessages, + recentHumanMessages: taskContextMessages.filter( + (message) => !message.is_bot_message, + ), lastHumanMessage, taskCreatedAt: task.created_at, }), @@ -154,7 +152,7 @@ export function buildPendingPairedTurn(args: { chatJid, timezone, turnOutputs, - recentMessages, + recentMessages: taskContextMessages, labeledRecentMessages, }), channel: resolveChannel(taskStatus), @@ -185,7 +183,9 @@ export function buildPendingPairedTurn(args: { chatJid, timezone, turnOutputs, - recentHumanMessages, + recentHumanMessages: taskContextMessages.filter( + (message) => !message.is_bot_message, + ), lastHumanMessage, taskCreatedAt: task.created_at, }), diff --git a/src/message-runtime-prompts.ts b/src/message-runtime-prompts.ts index 7f38ee4..a202d35 100644 --- a/src/message-runtime-prompts.ts +++ b/src/message-runtime-prompts.ts @@ -1,4 +1,5 @@ import { buildArbiterContextPrompt } from './arbiter-context.js'; +import { TASK_USER_CONTEXT_START_SKEW_MS } from './message-runtime-task-context.js'; import { formatMessages } from './router.js'; import type { NewMessage, @@ -14,7 +15,6 @@ const CARRIED_FORWARD_OWNER_FINAL_GUIDANCE = `System note: If you see a message beginning with "${CARRIED_FORWARD_OWNER_FINAL_MARKER}", treat it as background only. Do not repeat, continue, or answer that carried-forward final directly. Respond only to the latest human request and the current task.`; const ARBITER_TURN_OUTPUT_CONTEXT_LIMIT = 6; -const TASK_USER_CONTEXT_START_SKEW_MS = 5_000; function isImageAttachment(attachment: OutboundAttachment): boolean { if (attachment.mime?.toLowerCase().startsWith('image/')) return true; diff --git a/src/message-runtime-queue.ts b/src/message-runtime-queue.ts index 9cda40d..08f7cd2 100644 --- a/src/message-runtime-queue.ts +++ b/src/message-runtime-queue.ts @@ -1,4 +1,4 @@ -import { getPairedTurnOutputs, getRecentChatMessages } from './db.js'; +import { getPairedTurnOutputs } from './db.js'; import { logger } from './logger.js'; import { buildArbiterPromptForTask, @@ -21,6 +21,7 @@ import { resolveQueuedPairedTurnRole, resolveQueuedTurnRole, } from './message-runtime-rules.js'; +import { getTaskContextMessages } from './message-runtime-task-context.js'; import { claimPairedTurnExecution } from './paired-follow-up-scheduler.js'; import type { ExecuteTurnFn, @@ -67,7 +68,10 @@ function buildQueuedGroupTurnPrompt(args: { formatMessages: (messages: NewMessage[], timezone: string) => string; }): string { if (args.turnRole === 'arbiter' && args.currentTask) { - const recentMessages = getRecentChatMessages(args.chatJid, 20); + const recentMessages = getTaskContextMessages( + args.chatJid, + args.currentTask, + ); return buildArbiterPromptForTask({ task: args.currentTask, chatJid: args.chatJid, @@ -165,13 +169,13 @@ export async function runPendingPairedTurnIfNeeded(args: { return null; } - const recentMessages = getRecentChatMessages(chatJid, 20); - const recentHumanMessages = recentMessages.filter( + const taskContextMessages = getTaskContextMessages(chatJid, task); + const recentHumanMessages = taskContextMessages.filter( (message) => !message.is_bot_message, ); const labeledRecentMessages = args.labelPairedSenders( chatJid, - recentMessages, + taskContextMessages, ); const pendingTurn = buildPendingPairedTurn({ chatJid, @@ -255,9 +259,11 @@ export async function runQueuedGroupTurn(args: { }); currentTask = resolvedTask.task; if (resolvedTask.supersededTask) { - fallbackMessages = getRecentChatMessages(chatJid, 20).filter( - (message) => !message.is_bot_message, - ); + fallbackMessages = currentTask + ? getTaskContextMessages(chatJid, currentTask).filter( + (message) => !message.is_bot_message, + ) + : missedMessages; } } const taskStatus = currentTask?.status; diff --git a/src/message-runtime-task-context.test.ts b/src/message-runtime-task-context.test.ts new file mode 100644 index 0000000..151b8b2 --- /dev/null +++ b/src/message-runtime-task-context.test.ts @@ -0,0 +1,85 @@ +import { beforeEach, describe, expect, it } from 'vitest'; + +import { _initTestDatabase, storeChatMetadata, storeMessage } from './db.js'; +import { getTaskContextMessages } from './message-runtime-task-context.js'; +import type { NewMessage, PairedTask } from './types.js'; + +function makeMessage( + content: string, + timestamp: string, + overrides: Partial = {}, +): NewMessage { + return { + id: `msg-${content}`, + chat_jid: 'group@test', + sender: 'user@test', + sender_name: 'User', + content, + timestamp, + is_from_me: false, + is_bot_message: false, + ...overrides, + }; +} + +function makeTask(overrides: Partial = {}): PairedTask { + return { + id: 'task-context', + chat_jid: 'group@test', + group_folder: 'group', + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + owner_agent_type: 'codex', + reviewer_agent_type: 'claude-code', + arbiter_agent_type: 'codex', + title: null, + source_ref: null, + plan_notes: null, + review_requested_at: null, + round_trip_count: 1, + owner_failure_count: 0, + owner_step_done_streak: 0, + finalize_step_done_count: 0, + task_done_then_user_reopen_count: 0, + empty_step_done_streak: 0, + status: 'review_ready', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-04-20T00:00:00.000Z', + updated_at: '2026-04-20T00:30:00.000Z', + ...overrides, + }; +} + +describe('message runtime task context', () => { + beforeEach(() => { + _initTestDatabase(); + }); + + it('keeps the original task request even after it falls outside the recent 20 messages', () => { + storeChatMetadata( + 'group@test', + 'Test Group', + '2026-04-20T00:00:00.000Z', + 'discord', + true, + ); + storeMessage(makeMessage('원래 사용자 요청', '2026-04-20T00:00:01.000Z')); + for (let index = 0; index < 25; index += 1) { + storeMessage( + makeMessage( + `후속 노이즈 ${index}`, + `2026-04-20T00:${String(index + 2).padStart(2, '0')}:00.000Z`, + { id: `noise-${index}`, is_bot_message: index % 2 === 0 }, + ), + ); + } + + const messages = getTaskContextMessages('group@test', makeTask()); + + expect(messages.map((message) => message.content)).toContain( + '원래 사용자 요청', + ); + }); +}); diff --git a/src/message-runtime-task-context.ts b/src/message-runtime-task-context.ts new file mode 100644 index 0000000..9d2a066 --- /dev/null +++ b/src/message-runtime-task-context.ts @@ -0,0 +1,35 @@ +import { getMessagesSince, getRecentChatMessages } from './db.js'; +import type { NewMessage, PairedTask } from './types.js'; + +export const TASK_USER_CONTEXT_START_SKEW_MS = 5_000; +const TASK_CONTEXT_MESSAGE_LIMIT = 200; +const NO_BOT_PREFIX_FILTER = ''; + +function taskContextStartTimestamp(task: PairedTask): string | null { + const taskCreatedMs = Date.parse(task.created_at); + if (!Number.isFinite(taskCreatedMs)) { + return null; + } + return new Date( + taskCreatedMs - TASK_USER_CONTEXT_START_SKEW_MS, + ).toISOString(); +} + +export function getTaskContextMessages( + chatJid: string, + task: PairedTask, +): NewMessage[] { + const sinceTimestamp = taskContextStartTimestamp(task); + if (!sinceTimestamp) { + return getRecentChatMessages(chatJid, 20); + } + const taskMessages = getMessagesSince( + chatJid, + sinceTimestamp, + NO_BOT_PREFIX_FILTER, + TASK_CONTEXT_MESSAGE_LIMIT, + ); + return taskMessages.length > 0 + ? taskMessages + : getRecentChatMessages(chatJid, 20); +}