fix: make paired evidence loss visible (#202)

runners/agent-runner/src/output-protocol.ts

@@ -3,7 +3,10 @@ import path from 'path';
 
 import {
   extractImageTagPaths,
+  imageTagCaption,
+  missingImageTagCaption,
   normalizeAgentOutput,
+  splitImageTagPromptParts,
   type RunnerStructuredOutput,
   writeProtocolOutput,
 } from 'ejclaw-runners-shared';
@@ -76,36 +79,55 @@ export function buildMultimodalContent(
   text: string,
   log: LogFn,
 ): StreamContent {
-  const { cleanText, imagePaths } = extractImageTagPaths(text);
+  const { imagePaths } = extractImageTagPaths(text);
   if (imagePaths.length === 0) return text;
 
   const blocks: ContentBlock[] = [];
-  if (cleanText) {
-    blocks.push({ type: 'text', text: cleanText });
-  }
+  const pushText = (value: string) => {
+    const trimmed = value.trim();
+    if (trimmed) blocks.push({ type: 'text', text: trimmed });
+  };
+
+  for (const part of splitImageTagPromptParts(text)) {
+    if (part.type === 'text') {
+      pushText(part.text);
+      continue;
+    }
 
-  for (const imgPath of imagePaths) {
     try {
-      if (!fs.existsSync(imgPath)) {
-        log(`Image not found, skipping: ${imgPath}`);
+      if (!fs.existsSync(part.path)) {
+        log(`Image not found, skipping: ${part.path}`);
+        pushText(missingImageTagCaption(part, 'file not found'));
         continue;
       }
-      const data = fs.readFileSync(imgPath).toString('base64');
-      const ext = path.extname(imgPath).toLowerCase();
-      const mediaType = (MIME_TYPES[ext] || 'image/png') as
+      const ext = path.extname(part.path).toLowerCase();
+      const mediaType = MIME_TYPES[ext] as
         | 'image/jpeg'
         | 'image/png'
         | 'image/gif'
-        | 'image/webp';
+        | 'image/webp'
+        | undefined;
+      if (!mediaType) {
+        log(`Unsupported image type, skipping: ${part.path}`);
+        pushText(
+          missingImageTagCaption(
+            part,
+            `unsupported image type ${ext || 'unknown'}`,
+          ),
+        );
+        continue;
+      }
+      const data = fs.readFileSync(part.path).toString('base64');
+      pushText(imageTagCaption(part));
       blocks.push({
         type: 'image',
         source: { type: 'base64', media_type: mediaType, data },
       });
-      log(`Added image block: ${imgPath} (${mediaType})`);
+      log(`Added image block: ${part.path} (${mediaType})`);
     } catch (err) {
-      log(
-        `Failed to read image ${imgPath}: ${err instanceof Error ? err.message : String(err)}`,
-      );
+      const reason = err instanceof Error ? err.message : String(err);
+      log(`Failed to read image ${part.path}: ${reason}`);
+      pushText(missingImageTagCaption(part, `read failed: ${reason}`));
     }
   }
 

runners/agent-runner/test/output-protocol.test.ts

@@ -35,6 +35,10 @@ describe('agent-runner multimodal prompts', () => {
     expect(Array.isArray(content)).toBe(true);
     expect(content).toEqual([
       { type: 'text', text: '리뷰 증거' },
+      {
+        type: 'text',
+        text: 'Image evidence: settings-v0.1.92-deployed-390.png',
+      },
       {
         type: 'image',
         source: {
@@ -46,4 +50,48 @@ describe('agent-runner multimodal prompts', () => {
     ]);
     expect(logs).toContain(`Added image block: ${imagePath} (image/png)`);
   });
+
+  it('keeps missing image evidence visible in Claude prompts', () => {
+    const missingPath = path.join(
+      os.tmpdir(),
+      'ejclaw-missing-image-evidence.png',
+    );
+    const logs: string[] = [];
+
+    const content = buildMultimodalContent(
+      `리뷰 증거\n[Image: expected-render.png → ${missingPath}]`,
+      (message) => logs.push(message),
+    );
+
+    expect(content).toEqual([
+      { type: 'text', text: '리뷰 증거' },
+      {
+        type: 'text',
+        text: `[Image unavailable: expected-render.png → ${missingPath} — file not found]`,
+      },
+    ]);
+    expect(logs).toContain(`Image not found, skipping: ${missingPath}`);
+  });
+
+  it('keeps unsupported image evidence visible instead of mislabeled', () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ejclaw-bmp-image-'));
+    cleanupDirs.push(dir);
+    const imagePath = path.join(dir, 'settings.bmp');
+    fs.writeFileSync(imagePath, Buffer.from('BMunsupported'));
+    const logs: string[] = [];
+
+    const content = buildMultimodalContent(
+      `리뷰 증거\n[Image: settings.bmp → ${imagePath}]`,
+      (message) => logs.push(message),
+    );
+
+    expect(content).toEqual([
+      { type: 'text', text: '리뷰 증거' },
+      {
+        type: 'text',
+        text: `[Image unavailable: settings.bmp → ${imagePath} — unsupported image type .bmp]`,
+      },
+    ]);
+    expect(logs).toContain(`Unsupported image type, skipping: ${imagePath}`);
+  });
 });

runners/codex-runner/src/app-server-input.ts

@@ -1,27 +1,64 @@
 import fs from 'fs';
+import path from 'path';
 
-import { extractImageTagPaths } from 'ejclaw-runners-shared';
+import {
+  extractImageTagPaths,
+  imageTagCaption,
+  missingImageTagCaption,
+  splitImageTagPromptParts,
+} from 'ejclaw-runners-shared';
 
 import type { AppServerInputItem } from './app-server-client.js';
 
+const SUPPORTED_LOCAL_IMAGE_EXTENSIONS = new Set([
+  '.jpg',
+  '.jpeg',
+  '.png',
+  '.gif',
+  '.webp',
+]);
+
 export function parseAppServerInput(
   text: string,
   log: (message: string) => void = () => undefined,
 ): AppServerInputItem[] {
-  const { cleanText, imagePaths } = extractImageTagPaths(text);
+  const { imagePaths } = extractImageTagPaths(text);
   const input: AppServerInputItem[] = [];
+  const pushText = (value: string) => {
+    const trimmed = value.trim();
+    if (trimmed) input.push({ type: 'text', text: trimmed });
+  };
 
-  if (cleanText) {
-    input.push({ type: 'text', text: cleanText });
-  }
+  if (imagePaths.length > 0) {
+    for (const part of splitImageTagPromptParts(text)) {
+      if (part.type === 'text') {
+        pushText(part.text);
+        continue;
+      }
+      if (!fs.existsSync(part.path)) {
+        log(`Image not found, skipping: ${part.path}`);
+        pushText(missingImageTagCaption(part, 'file not found'));
+        continue;
+      }
 
-  for (const imgPath of imagePaths) {
-    if (fs.existsSync(imgPath)) {
-      input.push({ type: 'localImage', path: imgPath });
-      log(`Adding image input: ${imgPath}`);
-    } else {
-      log(`Image not found, skipping: ${imgPath}`);
+      const ext = path.extname(part.path).toLowerCase();
+      if (!SUPPORTED_LOCAL_IMAGE_EXTENSIONS.has(ext)) {
+        log(`Unsupported image type, skipping: ${part.path}`);
+        pushText(
+          missingImageTagCaption(
+            part,
+            `unsupported image type ${ext || 'unknown'}`,
+          ),
+        );
+        continue;
+      }
+
+      pushText(imageTagCaption(part));
+      input.push({ type: 'localImage', path: part.path });
+      log(`Adding image input: ${part.path}`);
     }
+  } else if (text) {
+    input.push({ type: 'text', text });
   }
 
   if (input.length === 0) {

runners/codex-runner/test/app-server-input.test.ts

@@ -34,8 +34,56 @@ describe('codex app-server input', () => {
 
     expect(input).toEqual([
       { type: 'text', text: '리뷰 증거' },
+      {
+        type: 'text',
+        text: 'Image evidence: settings-v0.1.92-deployed-390.png',
+      },
       { type: 'localImage', path: imagePath },
     ]);
     expect(logs).toContain(`Adding image input: ${imagePath}`);
   });
+
+  it('keeps missing image evidence visible in Codex prompts', () => {
+    const missingPath = path.join(
+      os.tmpdir(),
+      'ejclaw-missing-codex-image-evidence.png',
+    );
+    const logs: string[] = [];
+
+    const input = parseAppServerInput(
+      `리뷰 증거\n[Image: expected-render.png → ${missingPath}]`,
+      (message) => logs.push(message),
+    );
+
+    expect(input).toEqual([
+      { type: 'text', text: '리뷰 증거' },
+      {
+        type: 'text',
+        text: `[Image unavailable: expected-render.png → ${missingPath} — file not found]`,
+      },
+    ]);
+    expect(logs).toContain(`Image not found, skipping: ${missingPath}`);
+  });
+
+  it('keeps unsupported image evidence visible in Codex prompts', () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ejclaw-codex-bmp-'));
+    cleanupDirs.push(dir);
+    const imagePath = path.join(dir, 'settings.bmp');
+    fs.writeFileSync(imagePath, Buffer.from('BMunsupported'));
+    const logs: string[] = [];
+
+    const input = parseAppServerInput(
+      `리뷰 증거\n[Image: settings.bmp → ${imagePath}]`,
+      (message) => logs.push(message),
+    );
+
+    expect(input).toEqual([
+      { type: 'text', text: '리뷰 증거' },
+      {
+        type: 'text',
+        text: `[Image unavailable: settings.bmp → ${imagePath} — unsupported image type .bmp]`,
+      },
+    ]);
+    expect(logs).toContain(`Unsupported image type, skipping: ${imagePath}`);
+  });
 });

runners/shared/src/agent-protocol.ts

@@ -3,6 +3,7 @@ export const OUTPUT_END_MARKER = '---EJCLAW_OUTPUT_END---';
 
 export const IMAGE_TAG_RE =
   /\[Image:\s*(?:(?:[^\]\n]*?)\s*→\s*)?(\/[^\]\n]+)\]/g;
+const IMAGE_TAG_SEGMENT_RE = /\[Image:\s*(?:(.*?)\s*→\s*)?(\/[^\]\n]+)\]/g;
 const IMAGE_EXT_RE = /\.(png|jpe?g|gif|webp|bmp)$/i;
 const MARKDOWN_ABSOLUTE_LINK_RE = /!?\[[^\]\n]*\]\((\/[^)\n]+)\)/g;
 const MEDIA_TAG_RE =
@@ -63,6 +64,10 @@ function cloneImageTagPattern(): RegExp {
   return new RegExp(IMAGE_TAG_RE.source, IMAGE_TAG_RE.flags);
 }
 
+function cloneImageTagSegmentPattern(): RegExp {
+  return new RegExp(IMAGE_TAG_SEGMENT_RE.source, IMAGE_TAG_SEGMENT_RE.flags);
+}
+
 export function writeProtocolOutput<T>(
   output: T,
   writeLine: (line: string) => void = console.log,
@@ -87,6 +92,51 @@ export function extractImageTagPaths(text: string): {
   };
 }
 
+export type ImageTagPromptPart =
+  | { type: 'text'; text: string }
+  | { type: 'image'; label: string | null; path: string; raw: string };
+
+export function splitImageTagPromptParts(text: string): ImageTagPromptPart[] {
+  const imagePattern = cloneImageTagSegmentPattern();
+  const parts: ImageTagPromptPart[] = [];
+  let cursor = 0;
+
+  for (const match of text.matchAll(imagePattern)) {
+    const start = match.index ?? 0;
+    if (start > cursor) {
+      parts.push({ type: 'text', text: text.slice(cursor, start) });
+    }
+
+    const raw = match[0];
+    const label = match[1]?.trim() || null;
+    const imagePath = match[2].trim();
+    parts.push({ type: 'image', label, path: imagePath, raw });
+    cursor = start + raw.length;
+  }
+
+  if (cursor < text.length) {
+    parts.push({ type: 'text', text: text.slice(cursor) });
+  }
+
+  return parts.length > 0 ? parts : [{ type: 'text', text }];
+}
+
+export function imageTagCaption(
+  part: Extract<ImageTagPromptPart, { type: 'image' }>,
+): string {
+  return part.label
+    ? `Image evidence: ${part.label}`
+    : `Image evidence: ${part.path}`;
+}
+
+export function missingImageTagCaption(
+  part: Extract<ImageTagPromptPart, { type: 'image' }>,
+  reason: string,
+): string {
+  const label = part.label ? `${part.label} → ` : '';
+  return `[Image unavailable: ${label}${part.path} — ${reason}]`;
+}
+
 function attachmentName(filePath: string): string | undefined {
   return filePath.split(/[\\/]/).at(-1) || undefined;
 }

runners/shared/src/index.ts

@@ -41,15 +41,18 @@ export {
   extractMarkdownImageAttachments,
   extractMediaAttachments,
   extractImageTagPaths,
+  imageTagCaption,
   IMAGE_TAG_RE,
   IPC_CLOSE_SENTINEL,
   IPC_INPUT_SUBDIR,
   IPC_POLL_MS,
+  missingImageTagCaption,
   normalizeAgentOutput,
   normalizeEjclawStructuredOutput,
   normalizePublicTextOutput,
   OUTPUT_END_MARKER,
   OUTPUT_START_MARKER,
+  splitImageTagPromptParts,
   writeProtocolOutput,
   type NormalizedRunnerOutput,
   type NormalizedAgentOutput,

runners/shared/test/agent-protocol.test.ts

@@ -2,8 +2,11 @@ import { describe, expect, it } from 'vitest';
 
 import {
   extractImageTagPaths,
+  imageTagCaption,
+  missingImageTagCaption,
   normalizeEjclawStructuredOutput,
   normalizePublicTextOutput,
+  splitImageTagPromptParts,
   writeProtocolOutput,
 } from '../src/agent-protocol.js';
 
@@ -25,6 +28,37 @@ describe('shared agent protocol helpers', () => {
     });
   });
 
+  it('splits labeled image tags for multimodal prompt builders', () => {
+    expect(
+      splitImageTagPromptParts(
+        'before [Image: screenshot.png → /tmp/a.png] after',
+      ),
+    ).toEqual([
+      { type: 'text', text: 'before ' },
+      {
+        type: 'image',
+        label: 'screenshot.png',
+        path: '/tmp/a.png',
+        raw: '[Image: screenshot.png → /tmp/a.png]',
+      },
+      { type: 'text', text: ' after' },
+    ]);
+  });
+
+  it('formats image evidence captions and missing-image warnings', () => {
+    const part = {
+      type: 'image' as const,
+      label: 'screenshot.png',
+      path: '/tmp/a.png',
+      raw: '[Image: screenshot.png → /tmp/a.png]',
+    };
+
+    expect(imageTagCaption(part)).toBe('Image evidence: screenshot.png');
+    expect(missingImageTagCaption(part, 'file not found')).toBe(
+      '[Image unavailable: screenshot.png → /tmp/a.png — file not found]',
+    );
+  });
+
   it('normalizes plain text runner output as public text', () => {
     expect(normalizePublicTextOutput('DONE')).toEqual({
       result: 'DONE',