feat: add Discord image receiving for Claude Code and Codex agents
Download Discord image attachments to DATA_DIR/attachments/ and pass them as multimodal content blocks to both agent types: - Claude Code: base64 ImageBlockParam via Agent SDK MessageParam - Codex: localImage input block via app-server turn/start
This commit is contained in:
@@ -48,9 +48,13 @@ interface SessionsIndex {
|
|||||||
entries: SessionEntry[];
|
entries: SessionEntry[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ContentBlock =
|
||||||
|
| { type: 'text'; text: string }
|
||||||
|
| { type: 'image'; source: { type: 'base64'; media_type: string; data: string } };
|
||||||
|
|
||||||
interface SDKUserMessage {
|
interface SDKUserMessage {
|
||||||
type: 'user';
|
type: 'user';
|
||||||
message: { role: 'user'; content: string };
|
message: { role: 'user'; content: string | ContentBlock[] };
|
||||||
parent_tool_use_id: null;
|
parent_tool_use_id: null;
|
||||||
session_id: string;
|
session_id: string;
|
||||||
}
|
}
|
||||||
@@ -68,6 +72,51 @@ const MAX_TURNS = 100;
|
|||||||
const IPC_INPUT_CLOSE_SENTINEL = path.join(IPC_INPUT_DIR, '_close');
|
const IPC_INPUT_CLOSE_SENTINEL = path.join(IPC_INPUT_DIR, '_close');
|
||||||
const IPC_POLL_MS = 500;
|
const IPC_POLL_MS = 500;
|
||||||
|
|
||||||
|
const IMAGE_TAG_RE = /\[Image:\s*(\/[^\]]+)\]/g;
|
||||||
|
const MIME_TYPES: Record<string, string> = {
|
||||||
|
'.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
|
||||||
|
'.png': 'image/png', '.gif': 'image/gif', '.webp': 'image/webp',
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse [Image: /absolute/path] tags from text and build multimodal content.
|
||||||
|
* Returns a plain string if no images found, or ContentBlock[] with text + image blocks.
|
||||||
|
*/
|
||||||
|
function buildMultimodalContent(text: string): string | ContentBlock[] {
|
||||||
|
const imagePaths: string[] = [];
|
||||||
|
let match;
|
||||||
|
while ((match = IMAGE_TAG_RE.exec(text)) !== null) {
|
||||||
|
imagePaths.push(match[1].trim());
|
||||||
|
}
|
||||||
|
IMAGE_TAG_RE.lastIndex = 0; // reset regex state
|
||||||
|
|
||||||
|
if (imagePaths.length === 0) return text;
|
||||||
|
|
||||||
|
const blocks: ContentBlock[] = [];
|
||||||
|
const cleanText = text.replace(IMAGE_TAG_RE, '').trim();
|
||||||
|
if (cleanText) {
|
||||||
|
blocks.push({ type: 'text', text: cleanText });
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const imgPath of imagePaths) {
|
||||||
|
try {
|
||||||
|
if (!fs.existsSync(imgPath)) {
|
||||||
|
log(`Image not found, skipping: ${imgPath}`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const data = fs.readFileSync(imgPath).toString('base64');
|
||||||
|
const ext = path.extname(imgPath).toLowerCase();
|
||||||
|
const mediaType = MIME_TYPES[ext] || 'image/png';
|
||||||
|
blocks.push({ type: 'image', source: { type: 'base64', media_type: mediaType, data } });
|
||||||
|
log(`Added image block: ${imgPath} (${mediaType})`);
|
||||||
|
} catch (err) {
|
||||||
|
log(`Failed to read image ${imgPath}: ${err instanceof Error ? err.message : String(err)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return blocks.length > 0 ? blocks : text;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Push-based async iterable for streaming user messages to the SDK.
|
* Push-based async iterable for streaming user messages to the SDK.
|
||||||
* Keeps the iterable alive until end() is called, preventing isSingleUserTurn.
|
* Keeps the iterable alive until end() is called, preventing isSingleUserTurn.
|
||||||
@@ -78,9 +127,10 @@ class MessageStream {
|
|||||||
private done = false;
|
private done = false;
|
||||||
|
|
||||||
push(text: string): void {
|
push(text: string): void {
|
||||||
|
const content = buildMultimodalContent(text);
|
||||||
this.queue.push({
|
this.queue.push({
|
||||||
type: 'user',
|
type: 'user',
|
||||||
message: { role: 'user', content: text },
|
message: { role: 'user', content },
|
||||||
parent_tool_use_id: null,
|
parent_tool_use_id: null,
|
||||||
session_id: '',
|
session_id: '',
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -325,9 +325,35 @@ class CodexAppServer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async startTurn(threadId: string, text: string): Promise<string> {
|
async startTurn(threadId: string, text: string): Promise<string> {
|
||||||
|
// Parse [Image: /absolute/path] patterns and convert to multimodal input
|
||||||
|
const imagePattern = /\[Image:\s*(\/[^\]]+)\]/g;
|
||||||
|
const input: Array<Record<string, unknown>> = [];
|
||||||
|
const imagePaths: string[] = [];
|
||||||
|
let match;
|
||||||
|
while ((match = imagePattern.exec(text)) !== null) {
|
||||||
|
imagePaths.push(match[1].trim());
|
||||||
|
}
|
||||||
|
// Add text (with image tags stripped) as first input block
|
||||||
|
const cleanText = text.replace(imagePattern, '').trim();
|
||||||
|
if (cleanText) {
|
||||||
|
input.push({ type: 'text', text: cleanText, text_elements: [] });
|
||||||
|
}
|
||||||
|
// Add image input blocks
|
||||||
|
for (const imgPath of imagePaths) {
|
||||||
|
if (fs.existsSync(imgPath)) {
|
||||||
|
input.push({ type: 'localImage', path: imgPath });
|
||||||
|
log(`Adding image input: ${imgPath}`);
|
||||||
|
} else {
|
||||||
|
log(`Image not found, skipping: ${imgPath}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (input.length === 0) {
|
||||||
|
input.push({ type: 'text', text, text_elements: [] });
|
||||||
|
}
|
||||||
|
|
||||||
const params: Record<string, unknown> = {
|
const params: Record<string, unknown> = {
|
||||||
threadId,
|
threadId,
|
||||||
input: [{ type: 'text', text, text_elements: [] }],
|
input,
|
||||||
};
|
};
|
||||||
if (CODEX_EFFORT) params.effort = CODEX_EFFORT;
|
if (CODEX_EFFORT) params.effort = CODEX_EFFORT;
|
||||||
|
|
||||||
|
|||||||
@@ -1,3 +1,6 @@
|
|||||||
|
import fs from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
Attachment,
|
Attachment,
|
||||||
Client,
|
Client,
|
||||||
@@ -7,10 +10,30 @@ import {
|
|||||||
TextChannel,
|
TextChannel,
|
||||||
} from 'discord.js';
|
} from 'discord.js';
|
||||||
|
|
||||||
import { ASSISTANT_NAME, TRIGGER_PATTERN } from '../config.js';
|
import { ASSISTANT_NAME, DATA_DIR, TRIGGER_PATTERN } from '../config.js';
|
||||||
import { readEnvFile } from '../env.js';
|
import { readEnvFile } from '../env.js';
|
||||||
import { logger } from '../logger.js';
|
import { logger } from '../logger.js';
|
||||||
|
|
||||||
|
const ATTACHMENTS_DIR = path.join(DATA_DIR, 'attachments');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Download a Discord image attachment to local disk.
|
||||||
|
* Returns the absolute path to the saved file.
|
||||||
|
*/
|
||||||
|
async function downloadImage(att: Attachment): Promise<string> {
|
||||||
|
const res = await fetch(att.url);
|
||||||
|
if (!res.ok) throw new Error(`Download failed: ${res.status}`);
|
||||||
|
const buffer = Buffer.from(await res.arrayBuffer());
|
||||||
|
|
||||||
|
fs.mkdirSync(ATTACHMENTS_DIR, { recursive: true });
|
||||||
|
const ext = path.extname(att.name || 'image.png') || '.png';
|
||||||
|
const filename = `${Date.now()}-${att.id}${ext}`;
|
||||||
|
const filePath = path.join(ATTACHMENTS_DIR, filename);
|
||||||
|
fs.writeFileSync(filePath, buffer);
|
||||||
|
logger.info({ file: filename, size: buffer.length }, 'Image downloaded');
|
||||||
|
return filePath;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Transcribe an audio attachment via OpenAI Whisper API.
|
* Transcribe an audio attachment via OpenAI Whisper API.
|
||||||
* Returns the transcribed text, or a fallback placeholder on failure.
|
* Returns the transcribed text, or a fallback placeholder on failure.
|
||||||
@@ -158,7 +181,13 @@ export class DiscordChannel implements Channel {
|
|||||||
if (contentType.startsWith('audio/') && openaiKey) {
|
if (contentType.startsWith('audio/') && openaiKey) {
|
||||||
return transcribeAudio(att, openaiKey);
|
return transcribeAudio(att, openaiKey);
|
||||||
} else if (contentType.startsWith('image/')) {
|
} else if (contentType.startsWith('image/')) {
|
||||||
return `[Image: ${att.name || 'image'}]`;
|
try {
|
||||||
|
const imgPath = await downloadImage(att);
|
||||||
|
return `[Image: ${imgPath}]`;
|
||||||
|
} catch (err) {
|
||||||
|
logger.error({ err, file: att.name }, 'Image download failed');
|
||||||
|
return `[Image: ${att.name || 'image'} (download failed)]`;
|
||||||
|
}
|
||||||
} else if (contentType.startsWith('video/')) {
|
} else if (contentType.startsWith('video/')) {
|
||||||
return `[Video: ${att.name || 'video'}]`;
|
return `[Video: ${att.name || 'video'}]`;
|
||||||
} else if (contentType.startsWith('audio/')) {
|
} else if (contentType.startsWith('audio/')) {
|
||||||
|
|||||||
Reference in New Issue
Block a user