fix: load supported document attachments (#205)
This commit is contained in:
@@ -2,12 +2,12 @@ import fs from 'fs';
|
|||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
expandImagePromptReferences,
|
attachmentEvidenceCaption,
|
||||||
extractImageTagPaths,
|
expandPromptAttachmentReferences,
|
||||||
imageTagCaption,
|
missingAttachmentCaption,
|
||||||
missingImageTagCaption,
|
|
||||||
normalizeAgentOutput,
|
normalizeAgentOutput,
|
||||||
splitImageTagPromptParts,
|
splitPromptAttachmentParts,
|
||||||
|
type PromptAttachmentPart,
|
||||||
type RunnerStructuredOutput,
|
type RunnerStructuredOutput,
|
||||||
writeProtocolOutput,
|
writeProtocolOutput,
|
||||||
} from 'ejclaw-runners-shared';
|
} from 'ejclaw-runners-shared';
|
||||||
@@ -51,6 +51,21 @@ type ContentBlock =
|
|||||||
media_type: 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp';
|
media_type: 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp';
|
||||||
data: string;
|
data: string;
|
||||||
};
|
};
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: 'document';
|
||||||
|
title?: string | null;
|
||||||
|
source:
|
||||||
|
| {
|
||||||
|
type: 'base64';
|
||||||
|
media_type: 'application/pdf';
|
||||||
|
data: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: 'text';
|
||||||
|
media_type: 'text/plain';
|
||||||
|
data: string;
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
interface SDKUserMessage {
|
interface SDKUserMessage {
|
||||||
@@ -76,13 +91,72 @@ const MIME_TYPES: Record<string, string> = {
|
|||||||
'.webp': 'image/webp',
|
'.webp': 'image/webp',
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const TEXT_DOCUMENT_EXTENSIONS = new Set([
|
||||||
|
'.txt',
|
||||||
|
'.md',
|
||||||
|
'.markdown',
|
||||||
|
'.csv',
|
||||||
|
'.json',
|
||||||
|
'.log',
|
||||||
|
'.xml',
|
||||||
|
'.yaml',
|
||||||
|
'.yml',
|
||||||
|
'.toml',
|
||||||
|
'.ini',
|
||||||
|
'.cfg',
|
||||||
|
'.conf',
|
||||||
|
]);
|
||||||
|
|
||||||
|
function loadDocumentBlock(
|
||||||
|
part: Extract<PromptAttachmentPart, { type: 'attachment' }>,
|
||||||
|
log: LogFn,
|
||||||
|
): ContentBlock | string {
|
||||||
|
try {
|
||||||
|
if (!fs.existsSync(part.path)) {
|
||||||
|
log(`Document not found, skipping: ${part.path}`);
|
||||||
|
return missingAttachmentCaption(part, 'file not found');
|
||||||
|
}
|
||||||
|
|
||||||
|
const ext = path.extname(part.path).toLowerCase();
|
||||||
|
if (ext === '.pdf') {
|
||||||
|
const data = fs.readFileSync(part.path).toString('base64');
|
||||||
|
log(`Added document block: ${part.path} (application/pdf)`);
|
||||||
|
return {
|
||||||
|
type: 'document',
|
||||||
|
title: part.label ?? path.basename(part.path),
|
||||||
|
source: { type: 'base64', media_type: 'application/pdf', data },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (TEXT_DOCUMENT_EXTENSIONS.has(ext)) {
|
||||||
|
const data = fs.readFileSync(part.path, 'utf8');
|
||||||
|
log(`Added document block: ${part.path} (text/plain)`);
|
||||||
|
return {
|
||||||
|
type: 'document',
|
||||||
|
title: part.label ?? path.basename(part.path),
|
||||||
|
source: { type: 'text', media_type: 'text/plain', data },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
log(`Unsupported document type, skipping: ${part.path}`);
|
||||||
|
return missingAttachmentCaption(
|
||||||
|
part,
|
||||||
|
`unsupported document type ${ext || 'unknown'}`,
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
const reason = err instanceof Error ? err.message : String(err);
|
||||||
|
log(`Failed to read document ${part.path}: ${reason}`);
|
||||||
|
return missingAttachmentCaption(part, `read failed: ${reason}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export function buildMultimodalContent(
|
export function buildMultimodalContent(
|
||||||
text: string,
|
text: string,
|
||||||
log: LogFn,
|
log: LogFn,
|
||||||
): StreamContent {
|
): StreamContent {
|
||||||
const expandedText = expandImagePromptReferences(text);
|
const expandedText = expandPromptAttachmentReferences(text);
|
||||||
const { imagePaths } = extractImageTagPaths(expandedText);
|
const parts = splitPromptAttachmentParts(expandedText);
|
||||||
if (imagePaths.length === 0) return text;
|
if (!parts.some((part) => part.type === 'attachment')) return text;
|
||||||
|
|
||||||
const blocks: ContentBlock[] = [];
|
const blocks: ContentBlock[] = [];
|
||||||
const pushText = (value: string) => {
|
const pushText = (value: string) => {
|
||||||
@@ -90,16 +164,32 @@ export function buildMultimodalContent(
|
|||||||
if (trimmed) blocks.push({ type: 'text', text: trimmed });
|
if (trimmed) blocks.push({ type: 'text', text: trimmed });
|
||||||
};
|
};
|
||||||
|
|
||||||
for (const part of splitImageTagPromptParts(expandedText)) {
|
for (const part of parts) {
|
||||||
if (part.type === 'text') {
|
if (part.type === 'text') {
|
||||||
pushText(part.text);
|
pushText(part.text);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (part.kind === 'document') {
|
||||||
|
pushText(attachmentEvidenceCaption(part));
|
||||||
|
const block = loadDocumentBlock(part, log);
|
||||||
|
if (typeof block === 'string') {
|
||||||
|
pushText(block);
|
||||||
|
} else {
|
||||||
|
blocks.push(block);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (part.kind !== 'image') {
|
||||||
|
pushText(part.raw);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (!fs.existsSync(part.path)) {
|
if (!fs.existsSync(part.path)) {
|
||||||
log(`Image not found, skipping: ${part.path}`);
|
log(`Image not found, skipping: ${part.path}`);
|
||||||
pushText(missingImageTagCaption(part, 'file not found'));
|
pushText(missingAttachmentCaption(part, 'file not found'));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
const ext = path.extname(part.path).toLowerCase();
|
const ext = path.extname(part.path).toLowerCase();
|
||||||
@@ -112,7 +202,7 @@ export function buildMultimodalContent(
|
|||||||
if (!mediaType) {
|
if (!mediaType) {
|
||||||
log(`Unsupported image type, skipping: ${part.path}`);
|
log(`Unsupported image type, skipping: ${part.path}`);
|
||||||
pushText(
|
pushText(
|
||||||
missingImageTagCaption(
|
missingAttachmentCaption(
|
||||||
part,
|
part,
|
||||||
`unsupported image type ${ext || 'unknown'}`,
|
`unsupported image type ${ext || 'unknown'}`,
|
||||||
),
|
),
|
||||||
@@ -120,7 +210,7 @@ export function buildMultimodalContent(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
const data = fs.readFileSync(part.path).toString('base64');
|
const data = fs.readFileSync(part.path).toString('base64');
|
||||||
pushText(imageTagCaption(part));
|
pushText(attachmentEvidenceCaption(part));
|
||||||
blocks.push({
|
blocks.push({
|
||||||
type: 'image',
|
type: 'image',
|
||||||
source: { type: 'base64', media_type: mediaType, data },
|
source: { type: 'base64', media_type: mediaType, data },
|
||||||
@@ -129,7 +219,7 @@ export function buildMultimodalContent(
|
|||||||
} catch (err) {
|
} catch (err) {
|
||||||
const reason = err instanceof Error ? err.message : String(err);
|
const reason = err instanceof Error ? err.message : String(err);
|
||||||
log(`Failed to read image ${part.path}: ${reason}`);
|
log(`Failed to read image ${part.path}: ${reason}`);
|
||||||
pushText(missingImageTagCaption(part, `read failed: ${reason}`));
|
pushText(missingAttachmentCaption(part, `read failed: ${reason}`));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ const ONE_PIXEL_PNG = Buffer.from(
|
|||||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=',
|
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=',
|
||||||
'base64',
|
'base64',
|
||||||
);
|
);
|
||||||
|
const MINIMAL_PDF = Buffer.from('%PDF-1.4\n1 0 obj\n<<>>\nendobj\n%%EOF\n');
|
||||||
|
|
||||||
const cleanupDirs: string[] = [];
|
const cleanupDirs: string[] = [];
|
||||||
|
|
||||||
@@ -147,4 +148,62 @@ describe('agent-runner multimodal prompts', () => {
|
|||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('loads PDF file tags as Claude document blocks', () => {
|
||||||
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ejclaw-pdf-doc-'));
|
||||||
|
cleanupDirs.push(dir);
|
||||||
|
const pdfPath = path.join(dir, 'report.pdf');
|
||||||
|
fs.writeFileSync(pdfPath, MINIMAL_PDF);
|
||||||
|
const logs: string[] = [];
|
||||||
|
|
||||||
|
const content = buildMultimodalContent(
|
||||||
|
`검토 자료\n[File: report.pdf → ${pdfPath}]`,
|
||||||
|
(message) => logs.push(message),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(content).toEqual([
|
||||||
|
{ type: 'text', text: '검토 자료' },
|
||||||
|
{ type: 'text', text: 'Document evidence: report.pdf' },
|
||||||
|
{
|
||||||
|
type: 'document',
|
||||||
|
title: 'report.pdf',
|
||||||
|
source: {
|
||||||
|
type: 'base64',
|
||||||
|
media_type: 'application/pdf',
|
||||||
|
data: MINIMAL_PDF.toString('base64'),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
expect(logs).toContain(
|
||||||
|
`Added document block: ${pdfPath} (application/pdf)`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('loads text file tags as Claude text document blocks', () => {
|
||||||
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ejclaw-text-doc-'));
|
||||||
|
cleanupDirs.push(dir);
|
||||||
|
const textPath = path.join(dir, 'notes.md');
|
||||||
|
fs.writeFileSync(textPath, '# Notes\nEvidence text');
|
||||||
|
|
||||||
|
const content = buildMultimodalContent(
|
||||||
|
`검토 자료\nMEDIA:${textPath}`,
|
||||||
|
() => {
|
||||||
|
// no-op
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(content).toEqual([
|
||||||
|
{ type: 'text', text: '검토 자료' },
|
||||||
|
{ type: 'text', text: 'Document evidence: notes.md' },
|
||||||
|
{
|
||||||
|
type: 'document',
|
||||||
|
title: 'notes.md',
|
||||||
|
source: {
|
||||||
|
type: 'text',
|
||||||
|
media_type: 'text/plain',
|
||||||
|
data: '# Notes\nEvidence text',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import fs from 'fs';
|
|||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
expandImagePromptReferences,
|
expandPromptAttachmentReferences,
|
||||||
extractImageTagPaths,
|
extractImageTagPaths,
|
||||||
imageTagCaption,
|
imageTagCaption,
|
||||||
missingImageTagCaption,
|
missingImageTagCaption,
|
||||||
@@ -23,7 +23,7 @@ export function parseAppServerInput(
|
|||||||
text: string,
|
text: string,
|
||||||
log: (message: string) => void = () => undefined,
|
log: (message: string) => void = () => undefined,
|
||||||
): AppServerInputItem[] {
|
): AppServerInputItem[] {
|
||||||
const expandedText = expandImagePromptReferences(text);
|
const expandedText = expandPromptAttachmentReferences(text);
|
||||||
const { imagePaths } = extractImageTagPaths(expandedText);
|
const { imagePaths } = extractImageTagPaths(expandedText);
|
||||||
const input: AppServerInputItem[] = [];
|
const input: AppServerInputItem[] = [];
|
||||||
const pushText = (value: string) => {
|
const pushText = (value: string) => {
|
||||||
@@ -59,8 +59,8 @@ export function parseAppServerInput(
|
|||||||
input.push({ type: 'localImage', path: part.path });
|
input.push({ type: 'localImage', path: part.path });
|
||||||
log(`Adding image input: ${part.path}`);
|
log(`Adding image input: ${part.path}`);
|
||||||
}
|
}
|
||||||
} else if (text) {
|
} else if (expandedText) {
|
||||||
input.push({ type: 'text', text });
|
input.push({ type: 'text', text: expandedText });
|
||||||
}
|
}
|
||||||
|
|
||||||
if (input.length === 0) {
|
if (input.length === 0) {
|
||||||
|
|||||||
@@ -118,4 +118,12 @@ describe('codex app-server input', () => {
|
|||||||
{ type: 'localImage', path: imagePath },
|
{ type: 'localImage', path: imagePath },
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('keeps supported document references visible for Codex when no structured document input exists', () => {
|
||||||
|
const input = parseAppServerInput('증거\nMEDIA:/tmp/report.pdf');
|
||||||
|
|
||||||
|
expect(input).toEqual([
|
||||||
|
{ type: 'text', text: '증거\n[File: report.pdf → /tmp/report.pdf]' },
|
||||||
|
]);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -5,6 +5,10 @@ export const IMAGE_TAG_RE =
|
|||||||
/\[Image:\s*(?:(?:[^\]\n]*?)\s*→\s*)?(\/[^\]\n]+)\]/g;
|
/\[Image:\s*(?:(?:[^\]\n]*?)\s*→\s*)?(\/[^\]\n]+)\]/g;
|
||||||
const IMAGE_TAG_SEGMENT_RE = /\[Image:\s*(?:(.*?)\s*→\s*)?(\/[^\]\n]+)\]/g;
|
const IMAGE_TAG_SEGMENT_RE = /\[Image:\s*(?:(.*?)\s*→\s*)?(\/[^\]\n]+)\]/g;
|
||||||
const IMAGE_EXT_RE = /\.(png|jpe?g|gif|webp|bmp)$/i;
|
const IMAGE_EXT_RE = /\.(png|jpe?g|gif|webp|bmp)$/i;
|
||||||
|
const MODEL_DOCUMENT_EXT_RE =
|
||||||
|
/\.(pdf|txt|md|markdown|csv|json|log|xml|yaml|yml|toml|ini|cfg|conf)$/i;
|
||||||
|
const PROMPT_ATTACHMENT_TAG_SEGMENT_RE =
|
||||||
|
/\[(Image|File|Document|Attachment):\s*(?:(.*?)\s*→\s*)?(\/[^\]\n]+)\]/g;
|
||||||
const MARKDOWN_IMAGE_ABSOLUTE_LINK_RE = /!\[[^\]\n]*\]\((\/[^)\n]+)\)/g;
|
const MARKDOWN_IMAGE_ABSOLUTE_LINK_RE = /!\[[^\]\n]*\]\((\/[^)\n]+)\)/g;
|
||||||
const MEDIA_TAG_RE =
|
const MEDIA_TAG_RE =
|
||||||
/^[ \t]*MEDIA:\s*(?:"([^"\n]+)"|'([^'\n]+)'|`([^`\n]+)`|(\/\S+))[ \t]*$/gm;
|
/^[ \t]*MEDIA:\s*(?:"([^"\n]+)"|'([^'\n]+)'|`([^`\n]+)`|(\/\S+))[ \t]*$/gm;
|
||||||
@@ -122,10 +126,61 @@ export function expandImagePromptReferences(text: string): string {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isModelDocumentPath(filePath: string): boolean {
|
||||||
|
return MODEL_DOCUMENT_EXT_RE.test(filePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
function promptReferenceTag(filePath: string): 'Image' | 'File' | null {
|
||||||
|
if (IMAGE_EXT_RE.test(filePath)) return 'Image';
|
||||||
|
if (MODEL_DOCUMENT_EXT_RE.test(filePath)) return 'File';
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function expandPromptAttachmentReferences(text: string): string {
|
||||||
|
const codeSpans = fencedCodeSpans(text);
|
||||||
|
const withMediaAttachments = text.replace(
|
||||||
|
MEDIA_TAG_RE,
|
||||||
|
(full: string, doubleQuoted, singleQuoted, backticked, bare, offset) => {
|
||||||
|
if (isInsideSpans(offset, codeSpans)) return full;
|
||||||
|
const filePath = String(
|
||||||
|
doubleQuoted ?? singleQuoted ?? backticked ?? bare ?? '',
|
||||||
|
).trim();
|
||||||
|
if (!filePath.startsWith('/')) return full;
|
||||||
|
const tag = promptReferenceTag(filePath);
|
||||||
|
if (!tag) return full;
|
||||||
|
const name = attachmentName(filePath) ?? filePath;
|
||||||
|
return `[${tag}: ${name} → ${filePath}]`;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const markdownCodeSpans = fencedCodeSpans(withMediaAttachments);
|
||||||
|
return withMediaAttachments.replace(
|
||||||
|
MARKDOWN_IMAGE_ABSOLUTE_LINK_RE,
|
||||||
|
(full: string, rawPath: string, offset: number) => {
|
||||||
|
if (isInsideSpans(offset, markdownCodeSpans)) return full;
|
||||||
|
const filePath = rawPath.trim();
|
||||||
|
if (!IMAGE_EXT_RE.test(filePath)) return full;
|
||||||
|
const name = attachmentName(filePath) ?? filePath;
|
||||||
|
return `[Image: ${name} → ${filePath}]`;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export type ImageTagPromptPart =
|
export type ImageTagPromptPart =
|
||||||
| { type: 'text'; text: string }
|
| { type: 'text'; text: string }
|
||||||
| { type: 'image'; label: string | null; path: string; raw: string };
|
| { type: 'image'; label: string | null; path: string; raw: string };
|
||||||
|
|
||||||
|
export type PromptAttachmentPart =
|
||||||
|
| { type: 'text'; text: string }
|
||||||
|
| {
|
||||||
|
type: 'attachment';
|
||||||
|
kind: 'image' | 'document' | 'unsupported';
|
||||||
|
label: string | null;
|
||||||
|
path: string;
|
||||||
|
raw: string;
|
||||||
|
tag: 'Image' | 'File' | 'Document' | 'Attachment';
|
||||||
|
};
|
||||||
|
|
||||||
export function splitImageTagPromptParts(text: string): ImageTagPromptPart[] {
|
export function splitImageTagPromptParts(text: string): ImageTagPromptPart[] {
|
||||||
const imagePattern = cloneImageTagSegmentPattern();
|
const imagePattern = cloneImageTagSegmentPattern();
|
||||||
const parts: ImageTagPromptPart[] = [];
|
const parts: ImageTagPromptPart[] = [];
|
||||||
@@ -151,6 +206,49 @@ export function splitImageTagPromptParts(text: string): ImageTagPromptPart[] {
|
|||||||
return parts.length > 0 ? parts : [{ type: 'text', text }];
|
return parts.length > 0 ? parts : [{ type: 'text', text }];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function splitPromptAttachmentParts(
|
||||||
|
text: string,
|
||||||
|
): PromptAttachmentPart[] {
|
||||||
|
const pattern = new RegExp(
|
||||||
|
PROMPT_ATTACHMENT_TAG_SEGMENT_RE.source,
|
||||||
|
PROMPT_ATTACHMENT_TAG_SEGMENT_RE.flags,
|
||||||
|
);
|
||||||
|
const parts: PromptAttachmentPart[] = [];
|
||||||
|
let cursor = 0;
|
||||||
|
|
||||||
|
for (const match of text.matchAll(pattern)) {
|
||||||
|
const start = match.index ?? 0;
|
||||||
|
if (start > cursor) {
|
||||||
|
parts.push({ type: 'text', text: text.slice(cursor, start) });
|
||||||
|
}
|
||||||
|
|
||||||
|
const raw = match[0];
|
||||||
|
const tag = match[1] as 'Image' | 'File' | 'Document' | 'Attachment';
|
||||||
|
const label = match[2]?.trim() || null;
|
||||||
|
const filePath = match[3].trim();
|
||||||
|
const kind = IMAGE_EXT_RE.test(filePath)
|
||||||
|
? 'image'
|
||||||
|
: MODEL_DOCUMENT_EXT_RE.test(filePath)
|
||||||
|
? 'document'
|
||||||
|
: 'unsupported';
|
||||||
|
parts.push({
|
||||||
|
type: 'attachment',
|
||||||
|
kind,
|
||||||
|
label,
|
||||||
|
path: filePath,
|
||||||
|
raw,
|
||||||
|
tag,
|
||||||
|
});
|
||||||
|
cursor = start + raw.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cursor < text.length) {
|
||||||
|
parts.push({ type: 'text', text: text.slice(cursor) });
|
||||||
|
}
|
||||||
|
|
||||||
|
return parts.length > 0 ? parts : [{ type: 'text', text }];
|
||||||
|
}
|
||||||
|
|
||||||
export function imageTagCaption(
|
export function imageTagCaption(
|
||||||
part: Extract<ImageTagPromptPart, { type: 'image' }>,
|
part: Extract<ImageTagPromptPart, { type: 'image' }>,
|
||||||
): string {
|
): string {
|
||||||
@@ -167,6 +265,34 @@ export function missingImageTagCaption(
|
|||||||
return `[Image unavailable: ${label}${part.path} — ${reason}]`;
|
return `[Image unavailable: ${label}${part.path} — ${reason}]`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function attachmentEvidenceCaption(
|
||||||
|
part: Extract<PromptAttachmentPart, { type: 'attachment' }>,
|
||||||
|
): string {
|
||||||
|
const subject =
|
||||||
|
part.kind === 'image'
|
||||||
|
? 'Image'
|
||||||
|
: part.kind === 'document'
|
||||||
|
? 'Document'
|
||||||
|
: 'Attachment';
|
||||||
|
return part.label
|
||||||
|
? `${subject} evidence: ${part.label}`
|
||||||
|
: `${subject} evidence: ${part.path}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function missingAttachmentCaption(
|
||||||
|
part: Extract<PromptAttachmentPart, { type: 'attachment' }>,
|
||||||
|
reason: string,
|
||||||
|
): string {
|
||||||
|
const subject =
|
||||||
|
part.kind === 'image'
|
||||||
|
? 'Image'
|
||||||
|
: part.kind === 'document'
|
||||||
|
? 'Document'
|
||||||
|
: 'Attachment';
|
||||||
|
const label = part.label ? `${part.label} → ` : '';
|
||||||
|
return `[${subject} unavailable: ${label}${part.path} — ${reason}]`;
|
||||||
|
}
|
||||||
|
|
||||||
function attachmentName(filePath: string): string | undefined {
|
function attachmentName(filePath: string): string | undefined {
|
||||||
return filePath.split(/[\\/]/).at(-1) || undefined;
|
return filePath.split(/[\\/]/).at(-1) || undefined;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -38,6 +38,8 @@ export {
|
|||||||
type TaskContextMode,
|
type TaskContextMode,
|
||||||
} from './task-runtime.js';
|
} from './task-runtime.js';
|
||||||
export {
|
export {
|
||||||
|
attachmentEvidenceCaption,
|
||||||
|
expandPromptAttachmentReferences,
|
||||||
expandImagePromptReferences,
|
expandImagePromptReferences,
|
||||||
extractMarkdownImageAttachments,
|
extractMarkdownImageAttachments,
|
||||||
extractMediaAttachments,
|
extractMediaAttachments,
|
||||||
@@ -47,14 +49,18 @@ export {
|
|||||||
IPC_CLOSE_SENTINEL,
|
IPC_CLOSE_SENTINEL,
|
||||||
IPC_INPUT_SUBDIR,
|
IPC_INPUT_SUBDIR,
|
||||||
IPC_POLL_MS,
|
IPC_POLL_MS,
|
||||||
|
isModelDocumentPath,
|
||||||
|
missingAttachmentCaption,
|
||||||
missingImageTagCaption,
|
missingImageTagCaption,
|
||||||
normalizeAgentOutput,
|
normalizeAgentOutput,
|
||||||
normalizeEjclawStructuredOutput,
|
normalizeEjclawStructuredOutput,
|
||||||
normalizePublicTextOutput,
|
normalizePublicTextOutput,
|
||||||
|
splitPromptAttachmentParts,
|
||||||
OUTPUT_END_MARKER,
|
OUTPUT_END_MARKER,
|
||||||
OUTPUT_START_MARKER,
|
OUTPUT_START_MARKER,
|
||||||
splitImageTagPromptParts,
|
splitImageTagPromptParts,
|
||||||
writeProtocolOutput,
|
writeProtocolOutput,
|
||||||
|
type PromptAttachmentPart,
|
||||||
type NormalizedRunnerOutput,
|
type NormalizedRunnerOutput,
|
||||||
type NormalizedAgentOutput,
|
type NormalizedAgentOutput,
|
||||||
type RunnerOutputPhase,
|
type RunnerOutputPhase,
|
||||||
|
|||||||
@@ -1,12 +1,16 @@
|
|||||||
import { describe, expect, it } from 'vitest';
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
attachmentEvidenceCaption,
|
||||||
|
expandPromptAttachmentReferences,
|
||||||
expandImagePromptReferences,
|
expandImagePromptReferences,
|
||||||
extractImageTagPaths,
|
extractImageTagPaths,
|
||||||
imageTagCaption,
|
imageTagCaption,
|
||||||
|
missingAttachmentCaption,
|
||||||
missingImageTagCaption,
|
missingImageTagCaption,
|
||||||
normalizeEjclawStructuredOutput,
|
normalizeEjclawStructuredOutput,
|
||||||
normalizePublicTextOutput,
|
normalizePublicTextOutput,
|
||||||
|
splitPromptAttachmentParts,
|
||||||
splitImageTagPromptParts,
|
splitImageTagPromptParts,
|
||||||
writeProtocolOutput,
|
writeProtocolOutput,
|
||||||
} from '../src/agent-protocol.js';
|
} from '../src/agent-protocol.js';
|
||||||
@@ -77,6 +81,60 @@ describe('shared agent protocol helpers', () => {
|
|||||||
expect(expandImagePromptReferences(text)).toBe(text);
|
expect(expandImagePromptReferences(text)).toBe(text);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('expands supported MEDIA documents into file prompt tags', () => {
|
||||||
|
expect(
|
||||||
|
expandPromptAttachmentReferences('증거\nMEDIA:/tmp/report.pdf'),
|
||||||
|
).toBe('증거\n[File: report.pdf → /tmp/report.pdf]');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('splits image and document prompt attachments in order', () => {
|
||||||
|
expect(
|
||||||
|
splitPromptAttachmentParts(
|
||||||
|
'before [Image: render.png → /tmp/render.png] middle [File: report.pdf → /tmp/report.pdf] after',
|
||||||
|
),
|
||||||
|
).toEqual([
|
||||||
|
{ type: 'text', text: 'before ' },
|
||||||
|
{
|
||||||
|
type: 'attachment',
|
||||||
|
kind: 'image',
|
||||||
|
label: 'render.png',
|
||||||
|
path: '/tmp/render.png',
|
||||||
|
raw: '[Image: render.png → /tmp/render.png]',
|
||||||
|
tag: 'Image',
|
||||||
|
},
|
||||||
|
{ type: 'text', text: ' middle ' },
|
||||||
|
{
|
||||||
|
type: 'attachment',
|
||||||
|
kind: 'document',
|
||||||
|
label: 'report.pdf',
|
||||||
|
path: '/tmp/report.pdf',
|
||||||
|
raw: '[File: report.pdf → /tmp/report.pdf]',
|
||||||
|
tag: 'File',
|
||||||
|
},
|
||||||
|
{ type: 'text', text: ' after' },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('formats document evidence captions and missing-document warnings', () => {
|
||||||
|
const part = {
|
||||||
|
type: 'attachment' as const,
|
||||||
|
kind: 'document' as const,
|
||||||
|
label: 'report.pdf',
|
||||||
|
path: '/tmp/report.pdf',
|
||||||
|
raw: '[File: report.pdf → /tmp/report.pdf]',
|
||||||
|
tag: 'File' as const,
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(attachmentEvidenceCaption(part)).toBe(
|
||||||
|
'Document evidence: report.pdf',
|
||||||
|
);
|
||||||
|
expect(missingAttachmentCaption(part, 'file not found')).toBe(
|
||||||
|
'[Document unavailable: report.pdf → /tmp/report.pdf — file not found]',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('shared agent output normalization', () => {
|
||||||
it('normalizes plain text runner output as public text', () => {
|
it('normalizes plain text runner output as public text', () => {
|
||||||
expect(normalizePublicTextOutput('DONE')).toEqual({
|
expect(normalizePublicTextOutput('DONE')).toEqual({
|
||||||
result: 'DONE',
|
result: 'DONE',
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import fs from 'fs';
|
|||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
|
||||||
import { Attachment } from 'discord.js';
|
import { Attachment } from 'discord.js';
|
||||||
|
import { isModelDocumentPath } from 'ejclaw-runners-shared';
|
||||||
|
|
||||||
import { DATA_DIR } from '../config.js';
|
import { DATA_DIR } from '../config.js';
|
||||||
import { logger } from '../logger.js';
|
import { logger } from '../logger.js';
|
||||||
@@ -63,6 +64,15 @@ function isTextAttachment(att: Attachment, contentType: string): boolean {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isStructuredDocumentAttachment(
|
||||||
|
att: Attachment,
|
||||||
|
contentType: string,
|
||||||
|
): boolean {
|
||||||
|
if (contentType === 'application/pdf') return true;
|
||||||
|
if (contentType.startsWith('text/')) return true;
|
||||||
|
return isModelDocumentPath(att.name || '');
|
||||||
|
}
|
||||||
|
|
||||||
function formatAttachmentReference(
|
function formatAttachmentReference(
|
||||||
label: 'Audio' | 'File' | 'Image' | 'Video',
|
label: 'Audio' | 'File' | 'Image' | 'Video',
|
||||||
att: Attachment,
|
att: Attachment,
|
||||||
@@ -121,6 +131,9 @@ export async function describeDownloadedAttachment(
|
|||||||
attachmentDefaultExtension(contentType),
|
attachmentDefaultExtension(contentType),
|
||||||
);
|
);
|
||||||
const reference = formatAttachmentReference(label, att, filePath);
|
const reference = formatAttachmentReference(label, att, filePath);
|
||||||
|
if (label === 'File' && isStructuredDocumentAttachment(att, contentType)) {
|
||||||
|
if (!isTextAttachment(att, contentType)) return reference;
|
||||||
|
}
|
||||||
if (!isTextAttachment(att, contentType)) {
|
if (!isTextAttachment(att, contentType)) {
|
||||||
if (label === 'Image') return reference;
|
if (label === 'Image') return reference;
|
||||||
return formatUnavailableAttachmentReference(
|
return formatUnavailableAttachmentReference(
|
||||||
|
|||||||
@@ -681,7 +681,7 @@ describe('attachments', () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('stores file attachment with a visible unsupported marker', async () => {
|
it('stores PDF file attachment as a structured document reference', async () => {
|
||||||
const opts = createTestOpts();
|
const opts = createTestOpts();
|
||||||
const channel = new DiscordChannel('test-token', opts);
|
const channel = new DiscordChannel('test-token', opts);
|
||||||
await channel.connect();
|
await channel.connect();
|
||||||
@@ -707,9 +707,7 @@ describe('attachments', () => {
|
|||||||
expect(opts.onMessage).toHaveBeenCalledWith(
|
expect(opts.onMessage).toHaveBeenCalledWith(
|
||||||
'dc:1234567890123456',
|
'dc:1234567890123456',
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
content: expect.stringMatching(
|
content: expect.stringMatching(/^\[File: report\.pdf → .+\.pdf\]$/),
|
||||||
/^\[File unavailable: report\.pdf → .+\.pdf — application\/pdf is not loaded as structured model input\]$/,
|
|
||||||
),
|
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -170,7 +170,7 @@ describe('message-runtime-prompts output-only context', () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('makes non-image turn attachments visibly unavailable instead of implying inspection', () => {
|
it('carries supported document turn attachments into reviewer prompts as file inputs', () => {
|
||||||
const prompt = buildReviewerPendingPrompt({
|
const prompt = buildReviewerPendingPrompt({
|
||||||
chatJid: 'group@test',
|
chatJid: 'group@test',
|
||||||
timezone: 'UTC',
|
timezone: 'UTC',
|
||||||
@@ -192,9 +192,7 @@ describe('message-runtime-prompts output-only context', () => {
|
|||||||
taskCreatedAt: '2026-04-20T01:00:00.000Z',
|
taskCreatedAt: '2026-04-20T01:00:00.000Z',
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(prompt).toContain(
|
expect(prompt).toContain('[File: report.pdf → /tmp/report.pdf]');
|
||||||
'[Attachment unavailable: report.pdf → /tmp/report.pdf — application/pdf is not loaded as structured model input]',
|
|
||||||
);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it('includes current task user scope in reviewer pending prompts without pulling older human messages', () => {
|
it('includes current task user scope in reviewer pending prompts without pulling older human messages', () => {
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import { isModelDocumentPath } from 'ejclaw-runners-shared';
|
||||||
|
|
||||||
import { buildArbiterContextPrompt } from './arbiter-context.js';
|
import { buildArbiterContextPrompt } from './arbiter-context.js';
|
||||||
import { TASK_USER_CONTEXT_START_SKEW_MS } from './message-runtime-task-context.js';
|
import { TASK_USER_CONTEXT_START_SKEW_MS } from './message-runtime-task-context.js';
|
||||||
import { formatMessages } from './router.js';
|
import { formatMessages } from './router.js';
|
||||||
@@ -21,6 +23,12 @@ function isImageAttachment(attachment: OutboundAttachment): boolean {
|
|||||||
return /\.(?:png|jpe?g|gif|webp|bmp)$/i.test(attachment.path);
|
return /\.(?:png|jpe?g|gif|webp|bmp)$/i.test(attachment.path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isDocumentAttachment(attachment: OutboundAttachment): boolean {
|
||||||
|
const mime = attachment.mime?.toLowerCase();
|
||||||
|
if (mime === 'application/pdf' || mime?.startsWith('text/')) return true;
|
||||||
|
return isModelDocumentPath(attachment.path);
|
||||||
|
}
|
||||||
|
|
||||||
function attachmentLabel(attachment: OutboundAttachment): string {
|
function attachmentLabel(attachment: OutboundAttachment): string {
|
||||||
return attachment.name?.trim() || attachment.path.split('/').at(-1) || 'file';
|
return attachment.name?.trim() || attachment.path.split('/').at(-1) || 'file';
|
||||||
}
|
}
|
||||||
@@ -39,11 +47,15 @@ function formatTurnOutputAttachmentContext(
|
|||||||
if (!attachments?.length) return '';
|
if (!attachments?.length) return '';
|
||||||
const lines = attachments.map((attachment) => {
|
const lines = attachments.map((attachment) => {
|
||||||
const label = attachmentLabel(attachment);
|
const label = attachmentLabel(attachment);
|
||||||
return isImageAttachment(attachment)
|
if (isImageAttachment(attachment)) {
|
||||||
? `[Image: ${label} → ${attachment.path}]`
|
return `[Image: ${label} → ${attachment.path}]`;
|
||||||
: `[Attachment unavailable: ${label} → ${attachment.path} — ${nonImageAttachmentReason(
|
}
|
||||||
attachment,
|
if (isDocumentAttachment(attachment)) {
|
||||||
)}]`;
|
return `[File: ${label} → ${attachment.path}]`;
|
||||||
|
}
|
||||||
|
return `[Attachment unavailable: ${label} → ${attachment.path} — ${nonImageAttachmentReason(
|
||||||
|
attachment,
|
||||||
|
)}]`;
|
||||||
});
|
});
|
||||||
return `\n\nAttached evidence from this turn:\n${lines.join('\n')}`;
|
return `\n\nAttached evidence from this turn:\n${lines.join('\n')}`;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user