fix(discord): preserve fenced code blocks across 2000-char chunks
Replace the naive byte-slice chunk loop with a fence-aware chunkForDiscord() splitter: prefers newline boundaries, re-closes any open ``` fence at the end of a chunk and reopens it with the same language tag at the start of the next, and is surrogate-pair safe. Adds 10 unit tests for the seam, multi-fence documents, surrogate pairs, and single-line overflow. Co-Authored-By: Claude Opus 4 <noreply@anthropic.com>
This commit is contained in:
@@ -129,7 +129,11 @@ vi.mock('discord.js', () => {
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
import { DiscordChannel, DiscordChannelOpts } from './discord.js';
|
import {
|
||||||
|
DiscordChannel,
|
||||||
|
DiscordChannelOpts,
|
||||||
|
chunkForDiscord,
|
||||||
|
} from './discord.js';
|
||||||
import { logger } from '../logger.js';
|
import { logger } from '../logger.js';
|
||||||
|
|
||||||
// --- Test helpers ---
|
// --- Test helpers ---
|
||||||
@@ -1138,3 +1142,125 @@ describe('channel properties', () => {
|
|||||||
expect(channel.name).toBe('discord');
|
expect(channel.name).toBe('discord');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// --- chunkForDiscord ---
|
||||||
|
|
||||||
|
describe('chunkForDiscord', () => {
|
||||||
|
it('returns the input unchanged when within the limit', () => {
|
||||||
|
const text = 'short message';
|
||||||
|
expect(chunkForDiscord(text, 2000)).toEqual([text]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns empty array for empty input', () => {
|
||||||
|
expect(chunkForDiscord('', 2000)).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('splits on newline boundaries when possible', () => {
|
||||||
|
const text = 'aaa\nbbb\nccc\nddd';
|
||||||
|
const chunks = chunkForDiscord(text, 8);
|
||||||
|
// Every chunk must be ≤ 8 chars and joining them must rebuild the text
|
||||||
|
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(8);
|
||||||
|
expect(chunks.join('')).toBe(text);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reopens a fenced code block across chunks (with language)', () => {
|
||||||
|
const code = Array.from({ length: 200 }, (_, i) => `line ${i}`).join('\n');
|
||||||
|
const text = 'intro paragraph.\n\n```ts\n' + code + '\n```\nafter';
|
||||||
|
const chunks = chunkForDiscord(text, 500);
|
||||||
|
expect(chunks.length).toBeGreaterThan(1);
|
||||||
|
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(500);
|
||||||
|
// Every chunk must have a balanced number of ``` fences (so it renders
|
||||||
|
// as valid markdown on its own).
|
||||||
|
for (const c of chunks) {
|
||||||
|
const fenceMatches = c.match(/^```/gm) ?? [];
|
||||||
|
expect(fenceMatches.length % 2).toBe(0);
|
||||||
|
}
|
||||||
|
// The middle chunks must reopen the same language.
|
||||||
|
for (let i = 1; i < chunks.length; i++) {
|
||||||
|
// It should either start with the reopened fence, or never have been
|
||||||
|
// inside the fence at this point.
|
||||||
|
if (chunks[i].startsWith('```')) {
|
||||||
|
expect(chunks[i].startsWith('```ts\n')).toBe(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reopens a language-less fenced code block', () => {
|
||||||
|
const code = Array.from({ length: 200 }, (_, i) => `payload-${i}`).join(
|
||||||
|
'\n',
|
||||||
|
);
|
||||||
|
const text = '```\n' + code + '\n```';
|
||||||
|
const chunks = chunkForDiscord(text, 400);
|
||||||
|
expect(chunks.length).toBeGreaterThan(1);
|
||||||
|
for (const c of chunks) {
|
||||||
|
const fenceMatches = c.match(/^```/gm) ?? [];
|
||||||
|
expect(fenceMatches.length % 2).toBe(0);
|
||||||
|
}
|
||||||
|
// Joining back should rebuild a string whose un-fenced content equals
|
||||||
|
// the original code (allowing for extra fence pairs in the middle).
|
||||||
|
const reconstructed = chunks
|
||||||
|
.join('\n')
|
||||||
|
.replace(/```\n```\n/g, '') // drop reopen-close pairs at chunk seams
|
||||||
|
.replace(/```\n```$/g, '');
|
||||||
|
expect(reconstructed.includes('payload-0')).toBe(true);
|
||||||
|
expect(reconstructed.includes('payload-199')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('handles multiple fenced blocks in the same document', () => {
|
||||||
|
const blockA = Array.from({ length: 50 }, () => 'AAAA').join('\n');
|
||||||
|
const blockB = Array.from({ length: 50 }, () => 'BBBB').join('\n');
|
||||||
|
const text =
|
||||||
|
'intro\n```js\n' + blockA + '\n```\nbetween\n```py\n' + blockB + '\n```';
|
||||||
|
const chunks = chunkForDiscord(text, 200);
|
||||||
|
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(200);
|
||||||
|
for (const c of chunks) {
|
||||||
|
const fenceMatches = c.match(/^```/gm) ?? [];
|
||||||
|
expect(fenceMatches.length % 2).toBe(0);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not split a surrogate pair', () => {
|
||||||
|
// Emoji 🦄 is a surrogate pair (length 2 in JS).
|
||||||
|
const emoji = '🦄';
|
||||||
|
const text = emoji.repeat(50);
|
||||||
|
const chunks = chunkForDiscord(text, 9);
|
||||||
|
for (const c of chunks) {
|
||||||
|
// No chunk should start or end mid surrogate pair.
|
||||||
|
expect(c.charCodeAt(0) >= 0xdc00 && c.charCodeAt(0) <= 0xdfff).toBe(false);
|
||||||
|
const last = c.charCodeAt(c.length - 1);
|
||||||
|
expect(last >= 0xd800 && last <= 0xdbff).toBe(false);
|
||||||
|
}
|
||||||
|
expect(chunks.join('')).toBe(text);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back gracefully when a single line exceeds maxLength', () => {
|
||||||
|
const longLine = 'x'.repeat(5000);
|
||||||
|
const chunks = chunkForDiscord(longLine, 2000);
|
||||||
|
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(2000);
|
||||||
|
expect(chunks.join('')).toBe(longLine);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps a closing fence with its block when there is room', () => {
|
||||||
|
const text = '```ts\nshort code\n```';
|
||||||
|
const chunks = chunkForDiscord(text, 2000);
|
||||||
|
expect(chunks).toEqual([text]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('preserves total content across chunk seams (modulo reopen overhead)', () => {
|
||||||
|
const code = Array.from(
|
||||||
|
{ length: 80 },
|
||||||
|
(_, i) => `console.log(${i});`,
|
||||||
|
).join('\n');
|
||||||
|
const text = '```js\n' + code + '\n```';
|
||||||
|
const chunks = chunkForDiscord(text, 300);
|
||||||
|
// After removing the reopen-close fence pairs that the chunker inserts
|
||||||
|
// between chunks, the result should match the original text.
|
||||||
|
const stripped = chunks.join('\n').replace(/```\n```js\n/g, '');
|
||||||
|
// The first chunk still has the original opener and the last still has
|
||||||
|
// the original closer.
|
||||||
|
expect(stripped.startsWith('```js\n')).toBe(true);
|
||||||
|
expect(stripped.endsWith('```')).toBe(true);
|
||||||
|
expect(stripped.includes('console.log(0);')).toBe(true);
|
||||||
|
expect(stripped.includes('console.log(79);')).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -39,6 +39,113 @@ const DISCORD_OWNER_TOKEN_KEY = 'DISCORD_OWNER_BOT_TOKEN';
|
|||||||
const DISCORD_REVIEWER_TOKEN_KEY = 'DISCORD_REVIEWER_BOT_TOKEN';
|
const DISCORD_REVIEWER_TOKEN_KEY = 'DISCORD_REVIEWER_BOT_TOKEN';
|
||||||
const DISCORD_ARBITER_TOKEN_KEY = 'DISCORD_ARBITER_BOT_TOKEN';
|
const DISCORD_ARBITER_TOKEN_KEY = 'DISCORD_ARBITER_BOT_TOKEN';
|
||||||
|
|
||||||
|
// Matches a line that opens or closes a markdown fenced code block.
|
||||||
|
// Captures the fence marker (``` or ```` etc.) and the optional language tag.
|
||||||
|
const FENCE_LINE_RE = /^(`{3,})([^\s`]*)\s*$/;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Split `text` into chunks ≤ `maxLength` chars while keeping every chunk a
|
||||||
|
* valid standalone Markdown snippet for Discord. If a chunk has to end inside
|
||||||
|
* a ``` fenced code block, we append a closing fence to it and reopen the
|
||||||
|
* same fence (with the same language tag) at the start of the next chunk.
|
||||||
|
*
|
||||||
|
* Behaviour:
|
||||||
|
* - Prefers splitting at newline boundaries.
|
||||||
|
* - If a single line is itself longer than `maxLength`, falls back to a
|
||||||
|
* surrogate-pair-safe byte split for that line only.
|
||||||
|
* - Returns `[text]` unchanged when `text.length <= maxLength`.
|
||||||
|
*/
|
||||||
|
export function chunkForDiscord(text: string, maxLength: number): string[] {
|
||||||
|
if (text.length <= maxLength) return text.length > 0 ? [text] : [];
|
||||||
|
|
||||||
|
const lines = text.split('\n');
|
||||||
|
const chunks: string[] = [];
|
||||||
|
let current = '';
|
||||||
|
let openLang = ''; // language tag of currently open fence
|
||||||
|
let openMarker: string | null = null; // ``` or ```` etc; null when not in a fence
|
||||||
|
|
||||||
|
const flush = () => {
|
||||||
|
if (current === '') return;
|
||||||
|
if (openMarker !== null) {
|
||||||
|
if (!current.endsWith('\n')) current += '\n';
|
||||||
|
current += openMarker;
|
||||||
|
chunks.push(current);
|
||||||
|
current = openMarker + openLang + '\n';
|
||||||
|
} else {
|
||||||
|
chunks.push(current);
|
||||||
|
current = '';
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const hardSplitInto = (segment: string) => {
|
||||||
|
// Split `segment` across chunks at code-unit boundaries while avoiding
|
||||||
|
// splitting surrogate pairs. Used when a single source line exceeds the
|
||||||
|
// budget on its own.
|
||||||
|
let s = segment;
|
||||||
|
while (s.length > 0) {
|
||||||
|
const reserve = openMarker !== null ? openMarker.length + 1 : 0;
|
||||||
|
const room = maxLength - current.length - reserve;
|
||||||
|
if (room <= 0) {
|
||||||
|
flush();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let take = Math.min(s.length, room);
|
||||||
|
if (
|
||||||
|
take < s.length &&
|
||||||
|
take > 0 &&
|
||||||
|
s.charCodeAt(take - 1) >= 0xd800 &&
|
||||||
|
s.charCodeAt(take - 1) <= 0xdbff
|
||||||
|
) {
|
||||||
|
take--;
|
||||||
|
}
|
||||||
|
if (take <= 0) {
|
||||||
|
// Couldn't fit anything safely; flush and retry on an empty chunk.
|
||||||
|
flush();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
current += s.slice(0, take);
|
||||||
|
s = s.slice(take);
|
||||||
|
if (s.length > 0) flush();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
for (let li = 0; li < lines.length; li++) {
|
||||||
|
const isLast = li === lines.length - 1;
|
||||||
|
const line = lines[li];
|
||||||
|
const lineWithNL = isLast ? line : line + '\n';
|
||||||
|
const reserve = openMarker !== null ? openMarker.length + 1 : 0;
|
||||||
|
|
||||||
|
if (
|
||||||
|
current.length > 0 &&
|
||||||
|
current.length + lineWithNL.length + reserve > maxLength
|
||||||
|
) {
|
||||||
|
flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (lineWithNL.length + reserve > maxLength) {
|
||||||
|
hardSplitInto(lineWithNL);
|
||||||
|
} else {
|
||||||
|
current += lineWithNL;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update fence state AFTER appending the line so the very-next iteration
|
||||||
|
// accounts for it correctly.
|
||||||
|
const m = line.match(FENCE_LINE_RE);
|
||||||
|
if (m) {
|
||||||
|
if (openMarker === null) {
|
||||||
|
openMarker = m[1];
|
||||||
|
openLang = m[2] || '';
|
||||||
|
} else if (m[1] === openMarker) {
|
||||||
|
openMarker = null;
|
||||||
|
openLang = '';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (current.length > 0) chunks.push(current);
|
||||||
|
return chunks;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Wait for a pending transcription from the other service (poll cache file).
|
* Wait for a pending transcription from the other service (poll cache file).
|
||||||
* Returns the cached text, or null if timeout.
|
* Returns the cached text, or null if timeout.
|
||||||
@@ -467,10 +574,14 @@ export class DiscordChannel implements Channel {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Send text in chunks, attach first batch to the first chunk
|
// Send text in chunks. The chunker preserves fenced code blocks so a
|
||||||
|
// long ``` block that straddles the 2000-char boundary doesn't get
|
||||||
|
// split mid-fence (which Discord would render as literal backticks
|
||||||
|
// and break syntax for following chunks). It is also surrogate-pair
|
||||||
|
// safe (emoji etc.).
|
||||||
|
const messageChunks = chunkForDiscord(cleaned, MAX_LENGTH);
|
||||||
let fileBatchIndex = 0;
|
let fileBatchIndex = 0;
|
||||||
for (let i = 0; i < cleaned.length; i += MAX_LENGTH) {
|
for (const chunk of messageChunks) {
|
||||||
const chunk = cleaned.slice(i, i + MAX_LENGTH);
|
|
||||||
const batch = fileBatches[fileBatchIndex];
|
const batch = fileBatches[fileBatchIndex];
|
||||||
recordSentMessage(
|
recordSentMessage(
|
||||||
await textChannel.send({
|
await textChannel.send({
|
||||||
|
|||||||
Reference in New Issue
Block a user