diff --git a/src/channels/discord.test.ts b/src/channels/discord.test.ts index 277098f..35eb370 100644 --- a/src/channels/discord.test.ts +++ b/src/channels/discord.test.ts @@ -124,7 +124,7 @@ vi.mock('discord.js', () => { }; }); -import { DiscordChannel, DiscordChannelOpts } from './discord.js'; +import { DiscordChannel, DiscordChannelOpts, chunkForDiscord } from './discord.js'; import { registerChannel } from './registry.js'; import { getEnv } from '../env.js'; import { logger } from '../logger.js'; @@ -1046,3 +1046,125 @@ describe('DiscordChannel', () => { }); }); }); + +// --- chunkForDiscord --- + +describe('chunkForDiscord', () => { + it('returns the input unchanged when within the limit', () => { + const text = 'short message'; + expect(chunkForDiscord(text, 2000)).toEqual([text]); + }); + + it('returns empty array for empty input', () => { + expect(chunkForDiscord('', 2000)).toEqual([]); + }); + + it('splits on newline boundaries when possible', () => { + const text = 'aaa\nbbb\nccc\nddd'; + const chunks = chunkForDiscord(text, 8); + // Every chunk must be ≤ 8 chars and joining them must rebuild the text + for (const c of chunks) expect(c.length).toBeLessThanOrEqual(8); + expect(chunks.join('')).toBe(text); + }); + + it('reopens a fenced code block across chunks (with language)', () => { + const code = Array.from({ length: 200 }, (_, i) => `line ${i}`).join('\n'); + const text = 'intro paragraph.\n\n```ts\n' + code + '\n```\nafter'; + const chunks = chunkForDiscord(text, 500); + expect(chunks.length).toBeGreaterThan(1); + for (const c of chunks) expect(c.length).toBeLessThanOrEqual(500); + // Every chunk must have a balanced number of ``` fences (so it renders + // as valid markdown on its own). + for (const c of chunks) { + const fenceMatches = c.match(/^```/gm) ?? []; + expect(fenceMatches.length % 2).toBe(0); + } + // The middle chunks must reopen the same language. + for (let i = 1; i < chunks.length; i++) { + // It should either start with the reopened fence, or never have been + // inside the fence at this point. + if (chunks[i].startsWith('```')) { + expect(chunks[i].startsWith('```ts\n')).toBe(true); + } + } + }); + + it('reopens a language-less fenced code block', () => { + const code = Array.from({ length: 200 }, (_, i) => `payload-${i}`).join( + '\n', + ); + const text = '```\n' + code + '\n```'; + const chunks = chunkForDiscord(text, 400); + expect(chunks.length).toBeGreaterThan(1); + for (const c of chunks) { + const fenceMatches = c.match(/^```/gm) ?? []; + expect(fenceMatches.length % 2).toBe(0); + } + // Joining back should rebuild a string whose un-fenced content equals + // the original code (allowing for extra fence pairs in the middle). + const reconstructed = chunks + .join('\n') + .replace(/```\n```\n/g, '') // drop reopen-close pairs at chunk seams + .replace(/```\n```$/g, ''); + expect(reconstructed.includes('payload-0')).toBe(true); + expect(reconstructed.includes('payload-199')).toBe(true); + }); + + it('handles multiple fenced blocks in the same document', () => { + const blockA = Array.from({ length: 50 }, () => 'AAAA').join('\n'); + const blockB = Array.from({ length: 50 }, () => 'BBBB').join('\n'); + const text = + 'intro\n```js\n' + blockA + '\n```\nbetween\n```py\n' + blockB + '\n```'; + const chunks = chunkForDiscord(text, 200); + for (const c of chunks) expect(c.length).toBeLessThanOrEqual(200); + for (const c of chunks) { + const fenceMatches = c.match(/^```/gm) ?? []; + expect(fenceMatches.length % 2).toBe(0); + } + }); + + it('does not split a surrogate pair', () => { + // Emoji 🦄 is a surrogate pair (length 2 in JS). + const emoji = '🦄'; + const text = emoji.repeat(50); + const chunks = chunkForDiscord(text, 9); + for (const c of chunks) { + // No chunk should start or end mid surrogate pair. + expect(c.charCodeAt(0) >= 0xdc00 && c.charCodeAt(0) <= 0xdfff).toBe( + false, + ); + const last = c.charCodeAt(c.length - 1); + expect(last >= 0xd800 && last <= 0xdbff).toBe(false); + } + expect(chunks.join('')).toBe(text); + }); + + it('falls back gracefully when a single line exceeds maxLength', () => { + const longLine = 'x'.repeat(5000); + const chunks = chunkForDiscord(longLine, 2000); + for (const c of chunks) expect(c.length).toBeLessThanOrEqual(2000); + expect(chunks.join('')).toBe(longLine); + }); + + it('keeps a closing fence with its block when there is room', () => { + const text = '```ts\nshort code\n```'; + const chunks = chunkForDiscord(text, 2000); + expect(chunks).toEqual([text]); + }); + + it('preserves total content across chunk seams (modulo reopen overhead)', () => { + const code = + Array.from({ length: 80 }, (_, i) => `console.log(${i});`).join('\n'); + const text = '```js\n' + code + '\n```'; + const chunks = chunkForDiscord(text, 300); + // After removing the reopen-close fence pairs that the chunker inserts + // between chunks, the result should match the original text. + const stripped = chunks.join('\n').replace(/```\n```js\n/g, ''); + // The first chunk still has the original opener and the last still has + // the original closer. + expect(stripped.startsWith('```js\n')).toBe(true); + expect(stripped.endsWith('```')).toBe(true); + expect(stripped.includes('console.log(0);')).toBe(true); + expect(stripped.includes('console.log(79);')).toBe(true); + }); +}); diff --git a/src/channels/discord.ts b/src/channels/discord.ts index 9c66c7d..6d64a79 100644 --- a/src/channels/discord.ts +++ b/src/channels/discord.ts @@ -40,6 +40,114 @@ const DISCORD_ARBITER_TOKEN_KEY = 'DISCORD_ARBITER_BOT_TOKEN'; const IMAGE_EXTS = /\.(png|jpe?g|gif|webp|bmp)$/i; const MD_LINK_RE = /\[[^\]]*\]\((\/[^)]+)\)/g; +// Matches a line that opens or closes a markdown fenced code block. +// Captures the fence marker (``` or ```` etc.) and the optional language tag. +const FENCE_LINE_RE = /^(`{3,})([^\s`]*)\s*$/; + +/** + * Split `text` into chunks ≤ `maxLength` chars while keeping every chunk a + * valid standalone Markdown snippet for Discord. If a chunk has to end inside + * a ``` fenced code block, we append a closing fence to it and reopen the + * same fence (with the same language tag) at the start of the next chunk. + * + * Behaviour: + * - Prefers splitting at newline boundaries. + * - If a single line is itself longer than `maxLength`, falls back to a + * surrogate-pair-safe byte split for that line only. + * - Preserves the surrogate-pair safety guarantee of the previous splitter. + * - Returns `[text]` unchanged when `text.length <= maxLength`. + */ +export function chunkForDiscord(text: string, maxLength: number): string[] { + if (text.length <= maxLength) return text.length > 0 ? [text] : []; + + const lines = text.split('\n'); + const chunks: string[] = []; + let current = ''; + let openLang = ''; // language tag of currently open fence + let openMarker: string | null = null; // ``` or ```` etc; null when not in a fence + + const flush = () => { + if (current === '') return; + if (openMarker !== null) { + if (!current.endsWith('\n')) current += '\n'; + current += openMarker; + chunks.push(current); + current = openMarker + openLang + '\n'; + } else { + chunks.push(current); + current = ''; + } + }; + + const hardSplitInto = (segment: string) => { + // Split `segment` across chunks at code-unit boundaries while avoiding + // splitting surrogate pairs. Used when a single source line exceeds the + // budget on its own. + let s = segment; + while (s.length > 0) { + const reserve = openMarker !== null ? openMarker.length + 1 : 0; + const room = maxLength - current.length - reserve; + if (room <= 0) { + flush(); + continue; + } + let take = Math.min(s.length, room); + if ( + take < s.length && + take > 0 && + s.charCodeAt(take - 1) >= 0xd800 && + s.charCodeAt(take - 1) <= 0xdbff + ) { + take--; + } + if (take <= 0) { + // Couldn't fit anything safely; flush and retry on an empty chunk. + flush(); + continue; + } + current += s.slice(0, take); + s = s.slice(take); + if (s.length > 0) flush(); + } + }; + + for (let li = 0; li < lines.length; li++) { + const isLast = li === lines.length - 1; + const line = lines[li]; + const lineWithNL = isLast ? line : line + '\n'; + const reserve = openMarker !== null ? openMarker.length + 1 : 0; + + if ( + current.length > 0 && + current.length + lineWithNL.length + reserve > maxLength + ) { + flush(); + } + + if (lineWithNL.length + reserve > maxLength) { + hardSplitInto(lineWithNL); + } else { + current += lineWithNL; + } + + // Update fence state AFTER appending the line so the very-next iteration + // accounts for it correctly. + const m = line.match(FENCE_LINE_RE); + if (m) { + if (openMarker === null) { + openMarker = m[1]; + openLang = m[2] || ''; + } else if (m[1] === openMarker) { + openMarker = null; + openLang = ''; + } + } + } + + if (current.length > 0) chunks.push(current); + return chunks; +} + function extractMarkdownImageAttachments(text: string): { cleanText: string; attachments: OutboundAttachment[]; @@ -564,22 +672,13 @@ export class DiscordChannel implements Channel { ); } } else { - // Send text in chunks, attach first batch to the first chunk - // Use a helper that avoids splitting surrogate pairs (emoji, etc.) + // Send text in chunks. The chunker preserves fenced code blocks so a + // long ``` block that straddles the 2000-char boundary doesn't get + // split mid-fence (which Discord would render as literal backticks + // and break syntax for following chunks). + const messageChunks = chunkForDiscord(cleaned, MAX_LENGTH); let fileBatchIndex = 0; - for (let i = 0; i < cleaned.length; ) { - let end = Math.min(i + MAX_LENGTH, cleaned.length); - // If we'd split a surrogate pair, back up one code unit - if ( - end < cleaned.length && - end > i && - cleaned.charCodeAt(end - 1) >= 0xd800 && - cleaned.charCodeAt(end - 1) <= 0xdbff - ) { - end--; - } - const chunk = cleaned.slice(i, end); - i = end; + for (const chunk of messageChunks) { const batch = fileBatches[fileBatchIndex]; recordSentMessage( await textChannel.send({