diff --git a/src/channels/discord.test.ts b/src/channels/discord.test.ts index b1e4aaa..438a9aa 100644 --- a/src/channels/discord.test.ts +++ b/src/channels/discord.test.ts @@ -129,7 +129,11 @@ vi.mock('discord.js', () => { }; }); -import { DiscordChannel, DiscordChannelOpts } from './discord.js'; +import { + DiscordChannel, + DiscordChannelOpts, + chunkForDiscord, +} from './discord.js'; import { logger } from '../logger.js'; // --- Test helpers --- @@ -1138,3 +1142,125 @@ describe('channel properties', () => { expect(channel.name).toBe('discord'); }); }); + +// --- chunkForDiscord --- + +describe('chunkForDiscord', () => { + it('returns the input unchanged when within the limit', () => { + const text = 'short message'; + expect(chunkForDiscord(text, 2000)).toEqual([text]); + }); + + it('returns empty array for empty input', () => { + expect(chunkForDiscord('', 2000)).toEqual([]); + }); + + it('splits on newline boundaries when possible', () => { + const text = 'aaa\nbbb\nccc\nddd'; + const chunks = chunkForDiscord(text, 8); + // Every chunk must be ≤ 8 chars and joining them must rebuild the text + for (const c of chunks) expect(c.length).toBeLessThanOrEqual(8); + expect(chunks.join('')).toBe(text); + }); + + it('reopens a fenced code block across chunks (with language)', () => { + const code = Array.from({ length: 200 }, (_, i) => `line ${i}`).join('\n'); + const text = 'intro paragraph.\n\n```ts\n' + code + '\n```\nafter'; + const chunks = chunkForDiscord(text, 500); + expect(chunks.length).toBeGreaterThan(1); + for (const c of chunks) expect(c.length).toBeLessThanOrEqual(500); + // Every chunk must have a balanced number of ``` fences (so it renders + // as valid markdown on its own). + for (const c of chunks) { + const fenceMatches = c.match(/^```/gm) ?? []; + expect(fenceMatches.length % 2).toBe(0); + } + // The middle chunks must reopen the same language. + for (let i = 1; i < chunks.length; i++) { + // It should either start with the reopened fence, or never have been + // inside the fence at this point. + if (chunks[i].startsWith('```')) { + expect(chunks[i].startsWith('```ts\n')).toBe(true); + } + } + }); + + it('reopens a language-less fenced code block', () => { + const code = Array.from({ length: 200 }, (_, i) => `payload-${i}`).join( + '\n', + ); + const text = '```\n' + code + '\n```'; + const chunks = chunkForDiscord(text, 400); + expect(chunks.length).toBeGreaterThan(1); + for (const c of chunks) { + const fenceMatches = c.match(/^```/gm) ?? []; + expect(fenceMatches.length % 2).toBe(0); + } + // Joining back should rebuild a string whose un-fenced content equals + // the original code (allowing for extra fence pairs in the middle). + const reconstructed = chunks + .join('\n') + .replace(/```\n```\n/g, '') // drop reopen-close pairs at chunk seams + .replace(/```\n```$/g, ''); + expect(reconstructed.includes('payload-0')).toBe(true); + expect(reconstructed.includes('payload-199')).toBe(true); + }); + + it('handles multiple fenced blocks in the same document', () => { + const blockA = Array.from({ length: 50 }, () => 'AAAA').join('\n'); + const blockB = Array.from({ length: 50 }, () => 'BBBB').join('\n'); + const text = + 'intro\n```js\n' + blockA + '\n```\nbetween\n```py\n' + blockB + '\n```'; + const chunks = chunkForDiscord(text, 200); + for (const c of chunks) expect(c.length).toBeLessThanOrEqual(200); + for (const c of chunks) { + const fenceMatches = c.match(/^```/gm) ?? []; + expect(fenceMatches.length % 2).toBe(0); + } + }); + + it('does not split a surrogate pair', () => { + // Emoji 🦄 is a surrogate pair (length 2 in JS). + const emoji = '🦄'; + const text = emoji.repeat(50); + const chunks = chunkForDiscord(text, 9); + for (const c of chunks) { + // No chunk should start or end mid surrogate pair. + expect(c.charCodeAt(0) >= 0xdc00 && c.charCodeAt(0) <= 0xdfff).toBe(false); + const last = c.charCodeAt(c.length - 1); + expect(last >= 0xd800 && last <= 0xdbff).toBe(false); + } + expect(chunks.join('')).toBe(text); + }); + + it('falls back gracefully when a single line exceeds maxLength', () => { + const longLine = 'x'.repeat(5000); + const chunks = chunkForDiscord(longLine, 2000); + for (const c of chunks) expect(c.length).toBeLessThanOrEqual(2000); + expect(chunks.join('')).toBe(longLine); + }); + + it('keeps a closing fence with its block when there is room', () => { + const text = '```ts\nshort code\n```'; + const chunks = chunkForDiscord(text, 2000); + expect(chunks).toEqual([text]); + }); + + it('preserves total content across chunk seams (modulo reopen overhead)', () => { + const code = Array.from( + { length: 80 }, + (_, i) => `console.log(${i});`, + ).join('\n'); + const text = '```js\n' + code + '\n```'; + const chunks = chunkForDiscord(text, 300); + // After removing the reopen-close fence pairs that the chunker inserts + // between chunks, the result should match the original text. + const stripped = chunks.join('\n').replace(/```\n```js\n/g, ''); + // The first chunk still has the original opener and the last still has + // the original closer. + expect(stripped.startsWith('```js\n')).toBe(true); + expect(stripped.endsWith('```')).toBe(true); + expect(stripped.includes('console.log(0);')).toBe(true); + expect(stripped.includes('console.log(79);')).toBe(true); + }); +}); diff --git a/src/channels/discord.ts b/src/channels/discord.ts index 05b8fbf..8607998 100644 --- a/src/channels/discord.ts +++ b/src/channels/discord.ts @@ -39,6 +39,113 @@ const DISCORD_OWNER_TOKEN_KEY = 'DISCORD_OWNER_BOT_TOKEN'; const DISCORD_REVIEWER_TOKEN_KEY = 'DISCORD_REVIEWER_BOT_TOKEN'; const DISCORD_ARBITER_TOKEN_KEY = 'DISCORD_ARBITER_BOT_TOKEN'; +// Matches a line that opens or closes a markdown fenced code block. +// Captures the fence marker (``` or ```` etc.) and the optional language tag. +const FENCE_LINE_RE = /^(`{3,})([^\s`]*)\s*$/; + +/** + * Split `text` into chunks ≤ `maxLength` chars while keeping every chunk a + * valid standalone Markdown snippet for Discord. If a chunk has to end inside + * a ``` fenced code block, we append a closing fence to it and reopen the + * same fence (with the same language tag) at the start of the next chunk. + * + * Behaviour: + * - Prefers splitting at newline boundaries. + * - If a single line is itself longer than `maxLength`, falls back to a + * surrogate-pair-safe byte split for that line only. + * - Returns `[text]` unchanged when `text.length <= maxLength`. + */ +export function chunkForDiscord(text: string, maxLength: number): string[] { + if (text.length <= maxLength) return text.length > 0 ? [text] : []; + + const lines = text.split('\n'); + const chunks: string[] = []; + let current = ''; + let openLang = ''; // language tag of currently open fence + let openMarker: string | null = null; // ``` or ```` etc; null when not in a fence + + const flush = () => { + if (current === '') return; + if (openMarker !== null) { + if (!current.endsWith('\n')) current += '\n'; + current += openMarker; + chunks.push(current); + current = openMarker + openLang + '\n'; + } else { + chunks.push(current); + current = ''; + } + }; + + const hardSplitInto = (segment: string) => { + // Split `segment` across chunks at code-unit boundaries while avoiding + // splitting surrogate pairs. Used when a single source line exceeds the + // budget on its own. + let s = segment; + while (s.length > 0) { + const reserve = openMarker !== null ? openMarker.length + 1 : 0; + const room = maxLength - current.length - reserve; + if (room <= 0) { + flush(); + continue; + } + let take = Math.min(s.length, room); + if ( + take < s.length && + take > 0 && + s.charCodeAt(take - 1) >= 0xd800 && + s.charCodeAt(take - 1) <= 0xdbff + ) { + take--; + } + if (take <= 0) { + // Couldn't fit anything safely; flush and retry on an empty chunk. + flush(); + continue; + } + current += s.slice(0, take); + s = s.slice(take); + if (s.length > 0) flush(); + } + }; + + for (let li = 0; li < lines.length; li++) { + const isLast = li === lines.length - 1; + const line = lines[li]; + const lineWithNL = isLast ? line : line + '\n'; + const reserve = openMarker !== null ? openMarker.length + 1 : 0; + + if ( + current.length > 0 && + current.length + lineWithNL.length + reserve > maxLength + ) { + flush(); + } + + if (lineWithNL.length + reserve > maxLength) { + hardSplitInto(lineWithNL); + } else { + current += lineWithNL; + } + + // Update fence state AFTER appending the line so the very-next iteration + // accounts for it correctly. + const m = line.match(FENCE_LINE_RE); + if (m) { + if (openMarker === null) { + openMarker = m[1]; + openLang = m[2] || ''; + } else if (m[1] === openMarker) { + openMarker = null; + openLang = ''; + } + } + } + + if (current.length > 0) chunks.push(current); + return chunks; +} + /** * Wait for a pending transcription from the other service (poll cache file). * Returns the cached text, or null if timeout. @@ -467,10 +574,14 @@ export class DiscordChannel implements Channel { ); } } else { - // Send text in chunks, attach first batch to the first chunk + // Send text in chunks. The chunker preserves fenced code blocks so a + // long ``` block that straddles the 2000-char boundary doesn't get + // split mid-fence (which Discord would render as literal backticks + // and break syntax for following chunks). It is also surrogate-pair + // safe (emoji etc.). + const messageChunks = chunkForDiscord(cleaned, MAX_LENGTH); let fileBatchIndex = 0; - for (let i = 0; i < cleaned.length; i += MAX_LENGTH) { - const chunk = cleaned.slice(i, i + MAX_LENGTH); + for (const chunk of messageChunks) { const batch = fileBatches[fileBatchIndex]; recordSentMessage( await textChannel.send({