diff --git a/container/agent-runner/src/index.ts b/container/agent-runner/src/index.ts index 0be8353..6600a8b 100644 --- a/container/agent-runner/src/index.ts +++ b/container/agent-runner/src/index.ts @@ -48,9 +48,13 @@ interface SessionsIndex { entries: SessionEntry[]; } +type ContentBlock = + | { type: 'text'; text: string } + | { type: 'image'; source: { type: 'base64'; media_type: string; data: string } }; + interface SDKUserMessage { type: 'user'; - message: { role: 'user'; content: string }; + message: { role: 'user'; content: string | ContentBlock[] }; parent_tool_use_id: null; session_id: string; } @@ -68,6 +72,51 @@ const MAX_TURNS = 100; const IPC_INPUT_CLOSE_SENTINEL = path.join(IPC_INPUT_DIR, '_close'); const IPC_POLL_MS = 500; +const IMAGE_TAG_RE = /\[Image:\s*(\/[^\]]+)\]/g; +const MIME_TYPES: Record = { + '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', + '.png': 'image/png', '.gif': 'image/gif', '.webp': 'image/webp', +}; + +/** + * Parse [Image: /absolute/path] tags from text and build multimodal content. + * Returns a plain string if no images found, or ContentBlock[] with text + image blocks. + */ +function buildMultimodalContent(text: string): string | ContentBlock[] { + const imagePaths: string[] = []; + let match; + while ((match = IMAGE_TAG_RE.exec(text)) !== null) { + imagePaths.push(match[1].trim()); + } + IMAGE_TAG_RE.lastIndex = 0; // reset regex state + + if (imagePaths.length === 0) return text; + + const blocks: ContentBlock[] = []; + const cleanText = text.replace(IMAGE_TAG_RE, '').trim(); + if (cleanText) { + blocks.push({ type: 'text', text: cleanText }); + } + + for (const imgPath of imagePaths) { + try { + if (!fs.existsSync(imgPath)) { + log(`Image not found, skipping: ${imgPath}`); + continue; + } + const data = fs.readFileSync(imgPath).toString('base64'); + const ext = path.extname(imgPath).toLowerCase(); + const mediaType = MIME_TYPES[ext] || 'image/png'; + blocks.push({ type: 'image', source: { type: 'base64', media_type: mediaType, data } }); + log(`Added image block: ${imgPath} (${mediaType})`); + } catch (err) { + log(`Failed to read image ${imgPath}: ${err instanceof Error ? err.message : String(err)}`); + } + } + + return blocks.length > 0 ? blocks : text; +} + /** * Push-based async iterable for streaming user messages to the SDK. * Keeps the iterable alive until end() is called, preventing isSingleUserTurn. @@ -78,9 +127,10 @@ class MessageStream { private done = false; push(text: string): void { + const content = buildMultimodalContent(text); this.queue.push({ type: 'user', - message: { role: 'user', content: text }, + message: { role: 'user', content }, parent_tool_use_id: null, session_id: '', }); diff --git a/container/codex-runner/src/index.ts b/container/codex-runner/src/index.ts index eacdb5c..e46341f 100644 --- a/container/codex-runner/src/index.ts +++ b/container/codex-runner/src/index.ts @@ -325,9 +325,35 @@ class CodexAppServer { } async startTurn(threadId: string, text: string): Promise { + // Parse [Image: /absolute/path] patterns and convert to multimodal input + const imagePattern = /\[Image:\s*(\/[^\]]+)\]/g; + const input: Array> = []; + const imagePaths: string[] = []; + let match; + while ((match = imagePattern.exec(text)) !== null) { + imagePaths.push(match[1].trim()); + } + // Add text (with image tags stripped) as first input block + const cleanText = text.replace(imagePattern, '').trim(); + if (cleanText) { + input.push({ type: 'text', text: cleanText, text_elements: [] }); + } + // Add image input blocks + for (const imgPath of imagePaths) { + if (fs.existsSync(imgPath)) { + input.push({ type: 'localImage', path: imgPath }); + log(`Adding image input: ${imgPath}`); + } else { + log(`Image not found, skipping: ${imgPath}`); + } + } + if (input.length === 0) { + input.push({ type: 'text', text, text_elements: [] }); + } + const params: Record = { threadId, - input: [{ type: 'text', text, text_elements: [] }], + input, }; if (CODEX_EFFORT) params.effort = CODEX_EFFORT; diff --git a/src/channels/discord.ts b/src/channels/discord.ts index 39af5d2..5304b4f 100644 --- a/src/channels/discord.ts +++ b/src/channels/discord.ts @@ -1,3 +1,6 @@ +import fs from 'fs'; +import path from 'path'; + import { Attachment, Client, @@ -7,10 +10,30 @@ import { TextChannel, } from 'discord.js'; -import { ASSISTANT_NAME, TRIGGER_PATTERN } from '../config.js'; +import { ASSISTANT_NAME, DATA_DIR, TRIGGER_PATTERN } from '../config.js'; import { readEnvFile } from '../env.js'; import { logger } from '../logger.js'; +const ATTACHMENTS_DIR = path.join(DATA_DIR, 'attachments'); + +/** + * Download a Discord image attachment to local disk. + * Returns the absolute path to the saved file. + */ +async function downloadImage(att: Attachment): Promise { + const res = await fetch(att.url); + if (!res.ok) throw new Error(`Download failed: ${res.status}`); + const buffer = Buffer.from(await res.arrayBuffer()); + + fs.mkdirSync(ATTACHMENTS_DIR, { recursive: true }); + const ext = path.extname(att.name || 'image.png') || '.png'; + const filename = `${Date.now()}-${att.id}${ext}`; + const filePath = path.join(ATTACHMENTS_DIR, filename); + fs.writeFileSync(filePath, buffer); + logger.info({ file: filename, size: buffer.length }, 'Image downloaded'); + return filePath; +} + /** * Transcribe an audio attachment via OpenAI Whisper API. * Returns the transcribed text, or a fallback placeholder on failure. @@ -158,7 +181,13 @@ export class DiscordChannel implements Channel { if (contentType.startsWith('audio/') && openaiKey) { return transcribeAudio(att, openaiKey); } else if (contentType.startsWith('image/')) { - return `[Image: ${att.name || 'image'}]`; + try { + const imgPath = await downloadImage(att); + return `[Image: ${imgPath}]`; + } catch (err) { + logger.error({ err, file: att.name }, 'Image download failed'); + return `[Image: ${att.name || 'image'} (download failed)]`; + } } else if (contentType.startsWith('video/')) { return `[Video: ${att.name || 'video'}]`; } else if (contentType.startsWith('audio/')) {