From 65059719ce84ec271f90451dd7ae92c6417ca876 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 1 Jun 2026 01:20:28 +0900 Subject: [PATCH] fix(discord): preserve fenced code blocks across 2000-char chunks Long bot replies that exceeded Discord's 2000-char limit were split by a naive byte-slice loop. When the split fell inside a ```fenced code block the first chunk lost its closing fence and the next chunk lost its opener, so Discord rendered raw backticks and the remainder as headings or bold instead of code. Replace the slice loop with chunkForDiscord(), a fence-aware splitter that prefers newline boundaries, re-closes any open fence at the end of a chunk, and reopens it (with the same language tag) at the start of the next. Single oversize lines still fall back to a surrogate-pair-safe byte split. Adds 10 unit tests covering the seam, multi-fence documents, surrogate pairs, and single-line overflow. --- src/channels/discord.test.ts | 124 ++++++++++++++++++++++++++++++++- src/channels/discord.ts | 129 +++++++++++++++++++++++++++++++---- 2 files changed, 237 insertions(+), 16 deletions(-) diff --git a/src/channels/discord.test.ts b/src/channels/discord.test.ts index 277098f..35eb370 100644 --- a/src/channels/discord.test.ts +++ b/src/channels/discord.test.ts @@ -124,7 +124,7 @@ vi.mock('discord.js', () => { }; }); -import { DiscordChannel, DiscordChannelOpts } from './discord.js'; +import { DiscordChannel, DiscordChannelOpts, chunkForDiscord } from './discord.js'; import { registerChannel } from './registry.js'; import { getEnv } from '../env.js'; import { logger } from '../logger.js'; @@ -1046,3 +1046,125 @@ describe('DiscordChannel', () => { }); }); }); + +// --- chunkForDiscord --- + +describe('chunkForDiscord', () => { + it('returns the input unchanged when within the limit', () => { + const text = 'short message'; + expect(chunkForDiscord(text, 2000)).toEqual([text]); + }); + + it('returns empty array for empty input', () => { + expect(chunkForDiscord('', 2000)).toEqual([]); + }); + + it('splits on newline boundaries when possible', () => { + const text = 'aaa\nbbb\nccc\nddd'; + const chunks = chunkForDiscord(text, 8); + // Every chunk must be ≤ 8 chars and joining them must rebuild the text + for (const c of chunks) expect(c.length).toBeLessThanOrEqual(8); + expect(chunks.join('')).toBe(text); + }); + + it('reopens a fenced code block across chunks (with language)', () => { + const code = Array.from({ length: 200 }, (_, i) => `line ${i}`).join('\n'); + const text = 'intro paragraph.\n\n```ts\n' + code + '\n```\nafter'; + const chunks = chunkForDiscord(text, 500); + expect(chunks.length).toBeGreaterThan(1); + for (const c of chunks) expect(c.length).toBeLessThanOrEqual(500); + // Every chunk must have a balanced number of ``` fences (so it renders + // as valid markdown on its own). + for (const c of chunks) { + const fenceMatches = c.match(/^```/gm) ?? []; + expect(fenceMatches.length % 2).toBe(0); + } + // The middle chunks must reopen the same language. + for (let i = 1; i < chunks.length; i++) { + // It should either start with the reopened fence, or never have been + // inside the fence at this point. + if (chunks[i].startsWith('```')) { + expect(chunks[i].startsWith('```ts\n')).toBe(true); + } + } + }); + + it('reopens a language-less fenced code block', () => { + const code = Array.from({ length: 200 }, (_, i) => `payload-${i}`).join( + '\n', + ); + const text = '```\n' + code + '\n```'; + const chunks = chunkForDiscord(text, 400); + expect(chunks.length).toBeGreaterThan(1); + for (const c of chunks) { + const fenceMatches = c.match(/^```/gm) ?? []; + expect(fenceMatches.length % 2).toBe(0); + } + // Joining back should rebuild a string whose un-fenced content equals + // the original code (allowing for extra fence pairs in the middle). + const reconstructed = chunks + .join('\n') + .replace(/```\n```\n/g, '') // drop reopen-close pairs at chunk seams + .replace(/```\n```$/g, ''); + expect(reconstructed.includes('payload-0')).toBe(true); + expect(reconstructed.includes('payload-199')).toBe(true); + }); + + it('handles multiple fenced blocks in the same document', () => { + const blockA = Array.from({ length: 50 }, () => 'AAAA').join('\n'); + const blockB = Array.from({ length: 50 }, () => 'BBBB').join('\n'); + const text = + 'intro\n```js\n' + blockA + '\n```\nbetween\n```py\n' + blockB + '\n```'; + const chunks = chunkForDiscord(text, 200); + for (const c of chunks) expect(c.length).toBeLessThanOrEqual(200); + for (const c of chunks) { + const fenceMatches = c.match(/^```/gm) ?? []; + expect(fenceMatches.length % 2).toBe(0); + } + }); + + it('does not split a surrogate pair', () => { + // Emoji 🦄 is a surrogate pair (length 2 in JS). + const emoji = '🦄'; + const text = emoji.repeat(50); + const chunks = chunkForDiscord(text, 9); + for (const c of chunks) { + // No chunk should start or end mid surrogate pair. + expect(c.charCodeAt(0) >= 0xdc00 && c.charCodeAt(0) <= 0xdfff).toBe( + false, + ); + const last = c.charCodeAt(c.length - 1); + expect(last >= 0xd800 && last <= 0xdbff).toBe(false); + } + expect(chunks.join('')).toBe(text); + }); + + it('falls back gracefully when a single line exceeds maxLength', () => { + const longLine = 'x'.repeat(5000); + const chunks = chunkForDiscord(longLine, 2000); + for (const c of chunks) expect(c.length).toBeLessThanOrEqual(2000); + expect(chunks.join('')).toBe(longLine); + }); + + it('keeps a closing fence with its block when there is room', () => { + const text = '```ts\nshort code\n```'; + const chunks = chunkForDiscord(text, 2000); + expect(chunks).toEqual([text]); + }); + + it('preserves total content across chunk seams (modulo reopen overhead)', () => { + const code = + Array.from({ length: 80 }, (_, i) => `console.log(${i});`).join('\n'); + const text = '```js\n' + code + '\n```'; + const chunks = chunkForDiscord(text, 300); + // After removing the reopen-close fence pairs that the chunker inserts + // between chunks, the result should match the original text. + const stripped = chunks.join('\n').replace(/```\n```js\n/g, ''); + // The first chunk still has the original opener and the last still has + // the original closer. + expect(stripped.startsWith('```js\n')).toBe(true); + expect(stripped.endsWith('```')).toBe(true); + expect(stripped.includes('console.log(0);')).toBe(true); + expect(stripped.includes('console.log(79);')).toBe(true); + }); +}); diff --git a/src/channels/discord.ts b/src/channels/discord.ts index 9c66c7d..6d64a79 100644 --- a/src/channels/discord.ts +++ b/src/channels/discord.ts @@ -40,6 +40,114 @@ const DISCORD_ARBITER_TOKEN_KEY = 'DISCORD_ARBITER_BOT_TOKEN'; const IMAGE_EXTS = /\.(png|jpe?g|gif|webp|bmp)$/i; const MD_LINK_RE = /\[[^\]]*\]\((\/[^)]+)\)/g; +// Matches a line that opens or closes a markdown fenced code block. +// Captures the fence marker (``` or ```` etc.) and the optional language tag. +const FENCE_LINE_RE = /^(`{3,})([^\s`]*)\s*$/; + +/** + * Split `text` into chunks ≤ `maxLength` chars while keeping every chunk a + * valid standalone Markdown snippet for Discord. If a chunk has to end inside + * a ``` fenced code block, we append a closing fence to it and reopen the + * same fence (with the same language tag) at the start of the next chunk. + * + * Behaviour: + * - Prefers splitting at newline boundaries. + * - If a single line is itself longer than `maxLength`, falls back to a + * surrogate-pair-safe byte split for that line only. + * - Preserves the surrogate-pair safety guarantee of the previous splitter. + * - Returns `[text]` unchanged when `text.length <= maxLength`. + */ +export function chunkForDiscord(text: string, maxLength: number): string[] { + if (text.length <= maxLength) return text.length > 0 ? [text] : []; + + const lines = text.split('\n'); + const chunks: string[] = []; + let current = ''; + let openLang = ''; // language tag of currently open fence + let openMarker: string | null = null; // ``` or ```` etc; null when not in a fence + + const flush = () => { + if (current === '') return; + if (openMarker !== null) { + if (!current.endsWith('\n')) current += '\n'; + current += openMarker; + chunks.push(current); + current = openMarker + openLang + '\n'; + } else { + chunks.push(current); + current = ''; + } + }; + + const hardSplitInto = (segment: string) => { + // Split `segment` across chunks at code-unit boundaries while avoiding + // splitting surrogate pairs. Used when a single source line exceeds the + // budget on its own. + let s = segment; + while (s.length > 0) { + const reserve = openMarker !== null ? openMarker.length + 1 : 0; + const room = maxLength - current.length - reserve; + if (room <= 0) { + flush(); + continue; + } + let take = Math.min(s.length, room); + if ( + take < s.length && + take > 0 && + s.charCodeAt(take - 1) >= 0xd800 && + s.charCodeAt(take - 1) <= 0xdbff + ) { + take--; + } + if (take <= 0) { + // Couldn't fit anything safely; flush and retry on an empty chunk. + flush(); + continue; + } + current += s.slice(0, take); + s = s.slice(take); + if (s.length > 0) flush(); + } + }; + + for (let li = 0; li < lines.length; li++) { + const isLast = li === lines.length - 1; + const line = lines[li]; + const lineWithNL = isLast ? line : line + '\n'; + const reserve = openMarker !== null ? openMarker.length + 1 : 0; + + if ( + current.length > 0 && + current.length + lineWithNL.length + reserve > maxLength + ) { + flush(); + } + + if (lineWithNL.length + reserve > maxLength) { + hardSplitInto(lineWithNL); + } else { + current += lineWithNL; + } + + // Update fence state AFTER appending the line so the very-next iteration + // accounts for it correctly. + const m = line.match(FENCE_LINE_RE); + if (m) { + if (openMarker === null) { + openMarker = m[1]; + openLang = m[2] || ''; + } else if (m[1] === openMarker) { + openMarker = null; + openLang = ''; + } + } + } + + if (current.length > 0) chunks.push(current); + return chunks; +} + function extractMarkdownImageAttachments(text: string): { cleanText: string; attachments: OutboundAttachment[]; @@ -564,22 +672,13 @@ export class DiscordChannel implements Channel { ); } } else { - // Send text in chunks, attach first batch to the first chunk - // Use a helper that avoids splitting surrogate pairs (emoji, etc.) + // Send text in chunks. The chunker preserves fenced code blocks so a + // long ``` block that straddles the 2000-char boundary doesn't get + // split mid-fence (which Discord would render as literal backticks + // and break syntax for following chunks). + const messageChunks = chunkForDiscord(cleaned, MAX_LENGTH); let fileBatchIndex = 0; - for (let i = 0; i < cleaned.length; ) { - let end = Math.min(i + MAX_LENGTH, cleaned.length); - // If we'd split a surrogate pair, back up one code unit - if ( - end < cleaned.length && - end > i && - cleaned.charCodeAt(end - 1) >= 0xd800 && - cleaned.charCodeAt(end - 1) <= 0xdbff - ) { - end--; - } - const chunk = cleaned.slice(i, end); - i = end; + for (const chunk of messageChunks) { const batch = fileBatches[fileBatchIndex]; recordSentMessage( await textChannel.send({