claude-bot/EJClaw
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(discord): preserve fenced code blocks across 2000-char chunks

Browse Source 
Replace the naive byte-slice chunk loop with a fence-aware chunkForDiscord()
splitter: prefers newline boundaries, re-closes any open ``` fence at the end
of a chunk and reopens it with the same language tag at the start of the next,
and is surrogate-pair safe. Adds 10 unit tests for the seam, multi-fence
documents, surrogate pairs, and single-line overflow.

Co-Authored-By: Claude Opus 4 <noreply@anthropic.com>
This commit is contained in:
claude-bot
2026-06-01 17:27:44 +09:00
parent 5b16bb6694
commit 5f21c27c66
2 changed files with 241 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
src/channels/discord.test.ts
View File

@@ -129,7 +129,11 @@ vi.mock('discord.js', () => {
};
});
import { DiscordChannel, DiscordChannelOpts } from './discord.js';
import {
DiscordChannel,
DiscordChannelOpts,
chunkForDiscord,
} from './discord.js';
import { logger } from '../logger.js';
// --- Test helpers ---
@@ -1138,3 +1142,125 @@ describe('channel properties', () => {
expect(channel.name).toBe('discord');
});
});
// --- chunkForDiscord ---
describe('chunkForDiscord', () => {
it('returns the input unchanged when within the limit', () => {
const text = 'short message';
expect(chunkForDiscord(text, 2000)).toEqual([text]);
});
it('returns empty array for empty input', () => {
expect(chunkForDiscord('', 2000)).toEqual([]);
});
it('splits on newline boundaries when possible', () => {
const text = 'aaa\nbbb\nccc\nddd';
const chunks = chunkForDiscord(text, 8);
// Every chunk must be ≤ 8 chars and joining them must rebuild the text
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(8);
expect(chunks.join('')).toBe(text);
});
it('reopens a fenced code block across chunks (with language)', () => {
const code = Array.from({ length: 200 }, (_, i) => `line ${i}`).join('\n');
const text = 'intro paragraph.\n\n```ts\n' + code + '\n```\nafter';
const chunks = chunkForDiscord(text, 500);
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(500);
// Every chunk must have a balanced number of ``` fences (so it renders
// as valid markdown on its own).
for (const c of chunks) {
const fenceMatches = c.match(/^```/gm) ?? [];
expect(fenceMatches.length % 2).toBe(0);
}
// The middle chunks must reopen the same language.
for (let i = 1; i < chunks.length; i++) {
// It should either start with the reopened fence, or never have been
// inside the fence at this point.
if (chunks[i].startsWith('```')) {
expect(chunks[i].startsWith('```ts\n')).toBe(true);
}
}
});
it('reopens a language-less fenced code block', () => {
const code = Array.from({ length: 200 }, (_, i) => `payload-${i}`).join(
'\n',
);
const text = '```\n' + code + '\n```';
const chunks = chunkForDiscord(text, 400);
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) {
const fenceMatches = c.match(/^```/gm) ?? [];
expect(fenceMatches.length % 2).toBe(0);
}
// Joining back should rebuild a string whose un-fenced content equals
// the original code (allowing for extra fence pairs in the middle).
const reconstructed = chunks
.join('\n')
.replace(/```\n```\n/g, '') // drop reopen-close pairs at chunk seams
.replace(/```\n```$/g, '');
expect(reconstructed.includes('payload-0')).toBe(true);
expect(reconstructed.includes('payload-199')).toBe(true);
});
it('handles multiple fenced blocks in the same document', () => {
const blockA = Array.from({ length: 50 }, () => 'AAAA').join('\n');
const blockB = Array.from({ length: 50 }, () => 'BBBB').join('\n');
const text =
'intro\n```js\n' + blockA + '\n```\nbetween\n```py\n' + blockB + '\n```';
const chunks = chunkForDiscord(text, 200);
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(200);
for (const c of chunks) {
const fenceMatches = c.match(/^```/gm) ?? [];
expect(fenceMatches.length % 2).toBe(0);
}
});
it('does not split a surrogate pair', () => {
// Emoji 🦄 is a surrogate pair (length 2 in JS).
const emoji = '🦄';
const text = emoji.repeat(50);
const chunks = chunkForDiscord(text, 9);
for (const c of chunks) {
// No chunk should start or end mid surrogate pair.
expect(c.charCodeAt(0) >= 0xdc00 && c.charCodeAt(0) <= 0xdfff).toBe(false);
const last = c.charCodeAt(c.length - 1);
expect(last >= 0xd800 && last <= 0xdbff).toBe(false);
}
expect(chunks.join('')).toBe(text);
});
it('falls back gracefully when a single line exceeds maxLength', () => {
const longLine = 'x'.repeat(5000);
const chunks = chunkForDiscord(longLine, 2000);
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(2000);
expect(chunks.join('')).toBe(longLine);
});
it('keeps a closing fence with its block when there is room', () => {
const text = '```ts\nshort code\n```';
const chunks = chunkForDiscord(text, 2000);
expect(chunks).toEqual([text]);
});
it('preserves total content across chunk seams (modulo reopen overhead)', () => {
const code = Array.from(
{ length: 80 },
(_, i) => `console.log(${i});`,
).join('\n');
const text = '```js\n' + code + '\n```';
const chunks = chunkForDiscord(text, 300);
// After removing the reopen-close fence pairs that the chunker inserts
// between chunks, the result should match the original text.
const stripped = chunks.join('\n').replace(/```\n```js\n/g, '');
// The first chunk still has the original opener and the last still has
// the original closer.
expect(stripped.startsWith('```js\n')).toBe(true);
expect(stripped.endsWith('```')).toBe(true);
expect(stripped.includes('console.log(0);')).toBe(true);
expect(stripped.includes('console.log(79);')).toBe(true);
});
});

117
src/channels/discord.ts
View File

@@ -39,6 +39,113 @@ const DISCORD_OWNER_TOKEN_KEY = 'DISCORD_OWNER_BOT_TOKEN';
const DISCORD_REVIEWER_TOKEN_KEY = 'DISCORD_REVIEWER_BOT_TOKEN';
const DISCORD_ARBITER_TOKEN_KEY = 'DISCORD_ARBITER_BOT_TOKEN';
// Matches a line that opens or closes a markdown fenced code block.
// Captures the fence marker (``` or ```` etc.) and the optional language tag.
const FENCE_LINE_RE = /^(`{3,})([^\s`]*)\s*$/;
/**
* Split `text` into chunks ≤ `maxLength` chars while keeping every chunk a
* valid standalone Markdown snippet for Discord. If a chunk has to end inside
* a ``` fenced code block, we append a closing fence to it and reopen the
* same fence (with the same language tag) at the start of the next chunk.
*
* Behaviour:
* - Prefers splitting at newline boundaries.
* - If a single line is itself longer than `maxLength`, falls back to a
* surrogate-pair-safe byte split for that line only.
* - Returns `[text]` unchanged when `text.length <= maxLength`.
*/
export function chunkForDiscord(text: string, maxLength: number): string[] {
if (text.length <= maxLength) return text.length > 0 ? [text] : [];
const lines = text.split('\n');
const chunks: string[] = [];
let current = '';
let openLang = ''; // language tag of currently open fence
let openMarker: string | null = null; // ``` or ```` etc; null when not in a fence
const flush = () => {
if (current === '') return;
if (openMarker !== null) {
if (!current.endsWith('\n')) current += '\n';
current += openMarker;
chunks.push(current);
current = openMarker + openLang + '\n';
} else {
chunks.push(current);
current = '';
}
};
const hardSplitInto = (segment: string) => {
// Split `segment` across chunks at code-unit boundaries while avoiding
// splitting surrogate pairs. Used when a single source line exceeds the
// budget on its own.
let s = segment;
while (s.length > 0) {
const reserve = openMarker !== null ? openMarker.length + 1 : 0;
const room = maxLength - current.length - reserve;
if (room <= 0) {
flush();
continue;
}
let take = Math.min(s.length, room);
if (
take < s.length &&
take > 0 &&
s.charCodeAt(take - 1) >= 0xd800 &&
s.charCodeAt(take - 1) <= 0xdbff
) {
take--;
}
if (take <= 0) {
// Couldn't fit anything safely; flush and retry on an empty chunk.
flush();
continue;
}
current += s.slice(0, take);
s = s.slice(take);
if (s.length > 0) flush();
}
};
for (let li = 0; li < lines.length; li++) {
const isLast = li === lines.length - 1;
const line = lines[li];
const lineWithNL = isLast ? line : line + '\n';
const reserve = openMarker !== null ? openMarker.length + 1 : 0;
if (
current.length > 0 &&
current.length + lineWithNL.length + reserve > maxLength
) {
flush();
}
if (lineWithNL.length + reserve > maxLength) {
hardSplitInto(lineWithNL);
} else {
current += lineWithNL;
}
// Update fence state AFTER appending the line so the very-next iteration
// accounts for it correctly.
const m = line.match(FENCE_LINE_RE);
if (m) {
if (openMarker === null) {
openMarker = m[1];
openLang = m[2] || '';
} else if (m[1] === openMarker) {
openMarker = null;
openLang = '';
}
}
}
if (current.length > 0) chunks.push(current);
return chunks;
}
/**
* Wait for a pending transcription from the other service (poll cache file).
* Returns the cached text, or null if timeout.
@@ -467,10 +574,14 @@ export class DiscordChannel implements Channel {
);
}
} else {
// Send text in chunks, attach first batch to the first chunk
// Send text in chunks. The chunker preserves fenced code blocks so a
// long ``` block that straddles the 2000-char boundary doesn't get
// split mid-fence (which Discord would render as literal backticks
// and break syntax for following chunks). It is also surrogate-pair
// safe (emoji etc.).
const messageChunks = chunkForDiscord(cleaned, MAX_LENGTH);
let fileBatchIndex = 0;
for (let i = 0; i < cleaned.length; i += MAX_LENGTH) {
const chunk = cleaned.slice(i, i + MAX_LENGTH);
for (const chunk of messageChunks) {
const batch = fileBatches[fileBatchIndex];
recordSentMessage(
await textChannel.send({