fix: harden stripToolCallLeaks regex for CJK/non-ASCII garbage tokens
The previous regex used \w+ for descriptor tokens between the function
name and JSON body, which only matches ASCII word characters. Model
hallucinations often inject CJK/non-ASCII text (e.g. 彩神争霸) in this
position. Switch to (?:\s+[^\s{}]+)+ to match any whitespace-separated
non-brace tokens regardless of character set.
Also removes stale backward-compat re-exports from index.ts and updates
the test import for composeDashboardContent.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -272,6 +272,24 @@ describe('stripToolCallLeaks', () => {
|
|||||||
it('returns empty string for empty input', () => {
|
it('returns empty string for empty input', () => {
|
||||||
expect(stripToolCallLeaks('')).toBe('');
|
expect(stripToolCallLeaks('')).toBe('');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('strips tool-call leaks with CJK/non-ASCII garbage between tokens', () => {
|
||||||
|
const cjkLeak =
|
||||||
|
'to=functions.exec_command 彩神争霸大发快三 json code {"cmd":"printf \'noop\\n\'","yield_time_ms":1000,"max_output_tokens":20}';
|
||||||
|
expect(stripToolCallLeaks(cjkLeak)).toBe('');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('strips tool-call leaks with mixed non-ASCII and preserves surrounding text', () => {
|
||||||
|
const mixed =
|
||||||
|
'대기 중입니다.to=functions.exec_command 彩神争霸 json code {"cmd":"ls"}\n다음 작업';
|
||||||
|
expect(stripToolCallLeaks(mixed)).toBe('대기 중입니다.\n다음 작업');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('strips tool-call leaks with multiple non-ASCII words before JSON', () => {
|
||||||
|
const leak =
|
||||||
|
'to=functions.read_file données café résumé code {"path":"/tmp/test"}';
|
||||||
|
expect(stripToolCallLeaks(leak)).toBe('');
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('formatOutbound with tool-call leaks', () => {
|
describe('formatOutbound with tool-call leaks', () => {
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
import { describe, expect, it, vi } from 'vitest';
|
import { describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
composeDashboardContent,
|
|
||||||
editFormattedTrackedChannelMessage,
|
editFormattedTrackedChannelMessage,
|
||||||
sendFormattedChannelMessage,
|
sendFormattedChannelMessage,
|
||||||
sendFormattedTrackedChannelMessage,
|
sendFormattedTrackedChannelMessage,
|
||||||
} from './index.js';
|
} from './index.js';
|
||||||
|
import { composeDashboardContent } from './dashboard-render.js';
|
||||||
import { Channel } from './types.js';
|
import { Channel } from './types.js';
|
||||||
|
|
||||||
function makeChannel(overrides: Partial<Channel> = {}): Channel {
|
function makeChannel(overrides: Partial<Channel> = {}): Channel {
|
||||||
|
|||||||
@@ -70,10 +70,6 @@ import {
|
|||||||
stopTokenRefreshLoop,
|
stopTokenRefreshLoop,
|
||||||
} from './token-refresh.js';
|
} from './token-refresh.js';
|
||||||
|
|
||||||
// Re-export for backwards compatibility during refactor
|
|
||||||
export { escapeXml, formatMessages } from './router.js';
|
|
||||||
export { composeDashboardContent } from './dashboard-render.js';
|
|
||||||
|
|
||||||
// Token rotation is initialized lazily on first use or at startup below
|
// Token rotation is initialized lazily on first use or at startup below
|
||||||
|
|
||||||
export async function sendFormattedChannelMessage(
|
export async function sendFormattedChannelMessage(
|
||||||
|
|||||||
@@ -58,16 +58,20 @@ function redactSecrets(text: string): string {
|
|||||||
*
|
*
|
||||||
* When a model (especially Codex) enters a degenerate loop, it emits
|
* When a model (especially Codex) enters a degenerate loop, it emits
|
||||||
* tool-call intent as plaintext instead of actual tool calls. The format is:
|
* tool-call intent as plaintext instead of actual tool calls. The format is:
|
||||||
* to=functions.<name> <type> {<json>}
|
* to=functions.<name> <arbitrary tokens> {<json>}
|
||||||
* e.g. `to=functions.exec_command code {"cmd":"git status","yield_time_ms":1000}`
|
* e.g. `to=functions.exec_command code {"cmd":"git status","yield_time_ms":1000}`
|
||||||
*
|
*
|
||||||
|
* The tokens between the function name and JSON body can include non-ASCII
|
||||||
|
* characters (CJK, etc.) when the model hallucinates. The regex allows one or
|
||||||
|
* more non-whitespace descriptor tokens before the JSON brace.
|
||||||
|
*
|
||||||
* This function removes such fragments so they never reach Discord.
|
* This function removes such fragments so they never reach Discord.
|
||||||
*/
|
*/
|
||||||
export function stripToolCallLeaks(text: string): string {
|
export function stripToolCallLeaks(text: string): string {
|
||||||
// Match tool-call serialization: to=functions.<name> <word> {<json>}
|
// Match tool-call serialization: to=functions.<name> <descriptor tokens> {<json>}
|
||||||
// Handles up to one level of nested braces in the JSON body.
|
// Handles up to one level of nested braces in the JSON body.
|
||||||
const stripped = text.replace(
|
const stripped = text.replace(
|
||||||
/to=functions\.\w+\s+\w+\s+\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}/g,
|
/to=functions\.\w+(?:\s+[^\s{}]+)+\s+\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}/g,
|
||||||
'',
|
'',
|
||||||
);
|
);
|
||||||
// Collapse excessive blank lines left after stripping
|
// Collapse excessive blank lines left after stripping
|
||||||
|
|||||||
Reference in New Issue
Block a user