fix(discord): preserve fenced code blocks across 2000-char chunks

Long bot replies that exceeded Discord's 2000-char limit were split by a
naive byte-slice loop. When the split fell inside a ```fenced code block
the first chunk lost its closing fence and the next chunk lost its
opener, so Discord rendered raw backticks and the remainder as headings
or bold instead of code.

Replace the slice loop with chunkForDiscord(), a fence-aware splitter
that prefers newline boundaries, re-closes any open fence at the end of
a chunk, and reopens it (with the same language tag) at the start of the
next. Single oversize lines still fall back to a surrogate-pair-safe
byte split. Adds 10 unit tests covering the seam, multi-fence
documents, surrogate pairs, and single-line overflow.
This commit is contained in:
Codex
2026-06-01 01:20:28 +09:00
parent 32fbb00dd2
commit 65059719ce
2 changed files with 237 additions and 16 deletions

View File

@@ -124,7 +124,7 @@ vi.mock('discord.js', () => {
}; };
}); });
import { DiscordChannel, DiscordChannelOpts } from './discord.js'; import { DiscordChannel, DiscordChannelOpts, chunkForDiscord } from './discord.js';
import { registerChannel } from './registry.js'; import { registerChannel } from './registry.js';
import { getEnv } from '../env.js'; import { getEnv } from '../env.js';
import { logger } from '../logger.js'; import { logger } from '../logger.js';
@@ -1046,3 +1046,125 @@ describe('DiscordChannel', () => {
}); });
}); });
}); });
// --- chunkForDiscord ---
describe('chunkForDiscord', () => {
it('returns the input unchanged when within the limit', () => {
const text = 'short message';
expect(chunkForDiscord(text, 2000)).toEqual([text]);
});
it('returns empty array for empty input', () => {
expect(chunkForDiscord('', 2000)).toEqual([]);
});
it('splits on newline boundaries when possible', () => {
const text = 'aaa\nbbb\nccc\nddd';
const chunks = chunkForDiscord(text, 8);
// Every chunk must be ≤ 8 chars and joining them must rebuild the text
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(8);
expect(chunks.join('')).toBe(text);
});
it('reopens a fenced code block across chunks (with language)', () => {
const code = Array.from({ length: 200 }, (_, i) => `line ${i}`).join('\n');
const text = 'intro paragraph.\n\n```ts\n' + code + '\n```\nafter';
const chunks = chunkForDiscord(text, 500);
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(500);
// Every chunk must have a balanced number of ``` fences (so it renders
// as valid markdown on its own).
for (const c of chunks) {
const fenceMatches = c.match(/^```/gm) ?? [];
expect(fenceMatches.length % 2).toBe(0);
}
// The middle chunks must reopen the same language.
for (let i = 1; i < chunks.length; i++) {
// It should either start with the reopened fence, or never have been
// inside the fence at this point.
if (chunks[i].startsWith('```')) {
expect(chunks[i].startsWith('```ts\n')).toBe(true);
}
}
});
it('reopens a language-less fenced code block', () => {
const code = Array.from({ length: 200 }, (_, i) => `payload-${i}`).join(
'\n',
);
const text = '```\n' + code + '\n```';
const chunks = chunkForDiscord(text, 400);
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) {
const fenceMatches = c.match(/^```/gm) ?? [];
expect(fenceMatches.length % 2).toBe(0);
}
// Joining back should rebuild a string whose un-fenced content equals
// the original code (allowing for extra fence pairs in the middle).
const reconstructed = chunks
.join('\n')
.replace(/```\n```\n/g, '') // drop reopen-close pairs at chunk seams
.replace(/```\n```$/g, '');
expect(reconstructed.includes('payload-0')).toBe(true);
expect(reconstructed.includes('payload-199')).toBe(true);
});
it('handles multiple fenced blocks in the same document', () => {
const blockA = Array.from({ length: 50 }, () => 'AAAA').join('\n');
const blockB = Array.from({ length: 50 }, () => 'BBBB').join('\n');
const text =
'intro\n```js\n' + blockA + '\n```\nbetween\n```py\n' + blockB + '\n```';
const chunks = chunkForDiscord(text, 200);
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(200);
for (const c of chunks) {
const fenceMatches = c.match(/^```/gm) ?? [];
expect(fenceMatches.length % 2).toBe(0);
}
});
it('does not split a surrogate pair', () => {
// Emoji 🦄 is a surrogate pair (length 2 in JS).
const emoji = '🦄';
const text = emoji.repeat(50);
const chunks = chunkForDiscord(text, 9);
for (const c of chunks) {
// No chunk should start or end mid surrogate pair.
expect(c.charCodeAt(0) >= 0xdc00 && c.charCodeAt(0) <= 0xdfff).toBe(
false,
);
const last = c.charCodeAt(c.length - 1);
expect(last >= 0xd800 && last <= 0xdbff).toBe(false);
}
expect(chunks.join('')).toBe(text);
});
it('falls back gracefully when a single line exceeds maxLength', () => {
const longLine = 'x'.repeat(5000);
const chunks = chunkForDiscord(longLine, 2000);
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(2000);
expect(chunks.join('')).toBe(longLine);
});
it('keeps a closing fence with its block when there is room', () => {
const text = '```ts\nshort code\n```';
const chunks = chunkForDiscord(text, 2000);
expect(chunks).toEqual([text]);
});
it('preserves total content across chunk seams (modulo reopen overhead)', () => {
const code =
Array.from({ length: 80 }, (_, i) => `console.log(${i});`).join('\n');
const text = '```js\n' + code + '\n```';
const chunks = chunkForDiscord(text, 300);
// After removing the reopen-close fence pairs that the chunker inserts
// between chunks, the result should match the original text.
const stripped = chunks.join('\n').replace(/```\n```js\n/g, '');
// The first chunk still has the original opener and the last still has
// the original closer.
expect(stripped.startsWith('```js\n')).toBe(true);
expect(stripped.endsWith('```')).toBe(true);
expect(stripped.includes('console.log(0);')).toBe(true);
expect(stripped.includes('console.log(79);')).toBe(true);
});
});

View File

@@ -40,6 +40,114 @@ const DISCORD_ARBITER_TOKEN_KEY = 'DISCORD_ARBITER_BOT_TOKEN';
const IMAGE_EXTS = /\.(png|jpe?g|gif|webp|bmp)$/i; const IMAGE_EXTS = /\.(png|jpe?g|gif|webp|bmp)$/i;
const MD_LINK_RE = /\[[^\]]*\]\((\/[^)]+)\)/g; const MD_LINK_RE = /\[[^\]]*\]\((\/[^)]+)\)/g;
// Matches a line that opens or closes a markdown fenced code block.
// Captures the fence marker (``` or ```` etc.) and the optional language tag.
const FENCE_LINE_RE = /^(`{3,})([^\s`]*)\s*$/;
/**
* Split `text` into chunks ≤ `maxLength` chars while keeping every chunk a
* valid standalone Markdown snippet for Discord. If a chunk has to end inside
* a ``` fenced code block, we append a closing fence to it and reopen the
* same fence (with the same language tag) at the start of the next chunk.
*
* Behaviour:
* - Prefers splitting at newline boundaries.
* - If a single line is itself longer than `maxLength`, falls back to a
* surrogate-pair-safe byte split for that line only.
* - Preserves the surrogate-pair safety guarantee of the previous splitter.
* - Returns `[text]` unchanged when `text.length <= maxLength`.
*/
export function chunkForDiscord(text: string, maxLength: number): string[] {
if (text.length <= maxLength) return text.length > 0 ? [text] : [];
const lines = text.split('\n');
const chunks: string[] = [];
let current = '';
let openLang = ''; // language tag of currently open fence
let openMarker: string | null = null; // ``` or ```` etc; null when not in a fence
const flush = () => {
if (current === '') return;
if (openMarker !== null) {
if (!current.endsWith('\n')) current += '\n';
current += openMarker;
chunks.push(current);
current = openMarker + openLang + '\n';
} else {
chunks.push(current);
current = '';
}
};
const hardSplitInto = (segment: string) => {
// Split `segment` across chunks at code-unit boundaries while avoiding
// splitting surrogate pairs. Used when a single source line exceeds the
// budget on its own.
let s = segment;
while (s.length > 0) {
const reserve = openMarker !== null ? openMarker.length + 1 : 0;
const room = maxLength - current.length - reserve;
if (room <= 0) {
flush();
continue;
}
let take = Math.min(s.length, room);
if (
take < s.length &&
take > 0 &&
s.charCodeAt(take - 1) >= 0xd800 &&
s.charCodeAt(take - 1) <= 0xdbff
) {
take--;
}
if (take <= 0) {
// Couldn't fit anything safely; flush and retry on an empty chunk.
flush();
continue;
}
current += s.slice(0, take);
s = s.slice(take);
if (s.length > 0) flush();
}
};
for (let li = 0; li < lines.length; li++) {
const isLast = li === lines.length - 1;
const line = lines[li];
const lineWithNL = isLast ? line : line + '\n';
const reserve = openMarker !== null ? openMarker.length + 1 : 0;
if (
current.length > 0 &&
current.length + lineWithNL.length + reserve > maxLength
) {
flush();
}
if (lineWithNL.length + reserve > maxLength) {
hardSplitInto(lineWithNL);
} else {
current += lineWithNL;
}
// Update fence state AFTER appending the line so the very-next iteration
// accounts for it correctly.
const m = line.match(FENCE_LINE_RE);
if (m) {
if (openMarker === null) {
openMarker = m[1];
openLang = m[2] || '';
} else if (m[1] === openMarker) {
openMarker = null;
openLang = '';
}
}
}
if (current.length > 0) chunks.push(current);
return chunks;
}
function extractMarkdownImageAttachments(text: string): { function extractMarkdownImageAttachments(text: string): {
cleanText: string; cleanText: string;
attachments: OutboundAttachment[]; attachments: OutboundAttachment[];
@@ -564,22 +672,13 @@ export class DiscordChannel implements Channel {
); );
} }
} else { } else {
// Send text in chunks, attach first batch to the first chunk // Send text in chunks. The chunker preserves fenced code blocks so a
// Use a helper that avoids splitting surrogate pairs (emoji, etc.) // long ``` block that straddles the 2000-char boundary doesn't get
// split mid-fence (which Discord would render as literal backticks
// and break syntax for following chunks).
const messageChunks = chunkForDiscord(cleaned, MAX_LENGTH);
let fileBatchIndex = 0; let fileBatchIndex = 0;
for (let i = 0; i < cleaned.length; ) { for (const chunk of messageChunks) {
let end = Math.min(i + MAX_LENGTH, cleaned.length);
// If we'd split a surrogate pair, back up one code unit
if (
end < cleaned.length &&
end > i &&
cleaned.charCodeAt(end - 1) >= 0xd800 &&
cleaned.charCodeAt(end - 1) <= 0xdbff
) {
end--;
}
const chunk = cleaned.slice(i, end);
i = end;
const batch = fileBatches[fileBatchIndex]; const batch = fileBatches[fileBatchIndex];
recordSentMessage( recordSentMessage(
await textChannel.send({ await textChannel.send({