- When the Korean rewrite errors, the stream ends before it arrives, or a final must go out without a result event, append a Korean notice to the English text instead of sending it bare (content is kept: dropping a NEEDS_CONTEXT question would be worse than English). - A still-English rewrite is sent unchanged: the nudge tells the model to repeat only when the user explicitly asked for English. - Intermediate progress text cannot be rewritten mid-turn, so English progress (>= 20 Latin letters, Hangul < 10%) is dropped before it is flushed by tool_progress / tool results / the result event. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
252 lines
9.3 KiB
TypeScript
252 lines
9.3 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
|
|
const outputs: Array<{
|
|
status: string;
|
|
result: string | null;
|
|
phase?: string;
|
|
}> = [];
|
|
let scriptedTurns: Array<Array<Record<string, unknown>>> = [];
|
|
const promptsSeen: string[] = [];
|
|
|
|
vi.mock('@anthropic-ai/claude-agent-sdk', () => ({
|
|
query: ({ prompt }: { prompt: AsyncIterable<unknown> }) =>
|
|
(async function* () {
|
|
yield { type: 'system', subtype: 'init', session_id: 'sess-1' };
|
|
for await (const message of prompt) {
|
|
const content = (message as { message: { content: unknown } }).message
|
|
.content;
|
|
promptsSeen.push(
|
|
typeof content === 'string' ? content : JSON.stringify(content),
|
|
);
|
|
const turn = scriptedTurns.shift();
|
|
if (!turn) return; // CLI exits without answering
|
|
for (const event of turn) yield event;
|
|
}
|
|
})(),
|
|
}));
|
|
|
|
vi.mock('../src/claude-cli.js', () => ({
|
|
getClaudeCliPath: () => '/fake/claude',
|
|
}));
|
|
|
|
vi.mock('../src/output-protocol.js', async (importOriginal) => {
|
|
const actual =
|
|
await importOriginal<typeof import('../src/output-protocol.js')>();
|
|
return {
|
|
...actual,
|
|
writeOutput: (output: {
|
|
status: string;
|
|
result: string | null;
|
|
phase?: string;
|
|
}) => {
|
|
outputs.push({
|
|
status: output.status,
|
|
result: output.result,
|
|
phase: output.phase,
|
|
});
|
|
},
|
|
};
|
|
});
|
|
|
|
import { runClaudeQuery } from '../src/claude-query-runner.js';
|
|
import {
|
|
KOREAN_REWRITE_FAILED_NOTICE,
|
|
KOREAN_REWRITE_NUDGE,
|
|
isEnglishProgress,
|
|
isKoreanReplyTarget,
|
|
needsKoreanRewrite,
|
|
} from '../src/reply-language-guard.js';
|
|
|
|
const ENGLISH_FINAL =
|
|
'NEEDS_CONTEXT\n\nI accepted the Terms and entered the email and password, but Trimble now wants a verification code it emailed to you, so sign-in isn\'t finished yet.\n\n- I clicked "Accept" on the Trimble Offering Terms.\n- On the Trimble ID page, I rejected the cookie banner and entered your email.\n\n사용자 액션 아이템:\n1. Please send me the verification code.';
|
|
const KOREAN_FINAL =
|
|
'NEEDS_CONTEXT\n\n약관에 동의하고 이메일과 비밀번호를 입력했지만, Trimble이 이메일로 보낸 인증 코드를 요구해서 아직 로그인이 끝나지 않았습니다.';
|
|
|
|
// Real SDK assistant messages carry stop_reason only inside `message` (null
|
|
// while streaming); `topLevelEndTurn` exercises the runner's end_turn shortcut.
|
|
function assistant(text: string, topLevelEndTurn = false) {
|
|
return {
|
|
type: 'assistant',
|
|
...(topLevelEndTurn ? { stop_reason: 'end_turn' } : {}),
|
|
message: { content: [{ type: 'text', text }], stop_reason: null },
|
|
};
|
|
}
|
|
function result(text: string, subtype = 'success') {
|
|
return { type: 'result', subtype, result: text };
|
|
}
|
|
|
|
async function run() {
|
|
return runClaudeQuery({
|
|
prompt: '사용자 요청',
|
|
sessionId: undefined,
|
|
mcpServerPath: '/fake/ipc-mcp-stdio.js',
|
|
runnerInput: { prompt: '사용자 요청', groupFolder: 'g', chatJid: 'dc:1' },
|
|
sdkEnv: {},
|
|
reviewerRuntime: false,
|
|
claudeReadonlyReviewerRuntime: false,
|
|
claudeReadonlySandboxMode: null,
|
|
abortController: new AbortController(),
|
|
paths: {
|
|
groupDir: '/tmp/ejclaw-lang-guard-test',
|
|
groupFolder: 'g',
|
|
hostTasksDir: '/tmp/ejclaw-lang-guard-test/tasks',
|
|
ipcInputCloseSentinel: '/tmp/ejclaw-lang-guard-test/_close',
|
|
ipcInputDir: '/tmp/ejclaw-lang-guard-test/input',
|
|
workDir: '',
|
|
},
|
|
log: () => {},
|
|
});
|
|
}
|
|
|
|
// Text the host can post: finals and intermediate progress text (tool-progress
|
|
// timer labels like "Bash (3s)" are not prose and are left out).
|
|
const visible = () =>
|
|
outputs
|
|
.filter((o) => o.phase !== 'progress')
|
|
.map((o) => [o.phase ?? 'final', o.result]);
|
|
|
|
describe('needsKoreanRewrite', () => {
|
|
it('flags an English final answer', () => {
|
|
expect(needsKoreanRewrite(ENGLISH_FINAL)).toBe(true);
|
|
});
|
|
|
|
it('does not let Korean boilerplate lines hide an English answer', () => {
|
|
const english =
|
|
'TASK_DONE\n\nRelease 2.4.0 is out. The importer now keeps section planes, scene tabs and layer visibility when opening older files. Shadows render faster on machines without a graphics card, and the measure tool snaps to arc centers.\n\n사용자 액션 아이템: 없음\n리뷰 불필요: 요청받은 문구를 그대로 전달했을 뿐 변경 사항이 없습니다.';
|
|
expect(needsKoreanRewrite(english)).toBe(true);
|
|
});
|
|
|
|
it('keeps Korean answers full of identifiers, code and links', () => {
|
|
const korean =
|
|
'TASK_DONE\n\nv0.3.5 배포 완료했습니다. `src/installer-rp/pack.ts`의 `electron-builder` 설정을 고쳤습니다.\n\n```bash\nbun run build && git push origin main --follow-tags\n```\n릴리즈: https://git.tkrmagid.kr/tkrmagid/minecraft-resourcepack/releases/tag/v0.3.5\n- origin/main = 9efd4a6, CI green, assets uploaded';
|
|
expect(needsKoreanRewrite(korean)).toBe(false);
|
|
});
|
|
|
|
it('ignores short text, JSON and host error banners', () => {
|
|
expect(needsKoreanRewrite('TASK_DONE\n\nDone.')).toBe(false);
|
|
expect(needsKoreanRewrite(`{"answer": "${'x'.repeat(300)}"}`)).toBe(false);
|
|
expect(
|
|
needsKoreanRewrite(
|
|
`API Error: Opus 5.5's safeguards flagged this message. ${'word '.repeat(60)}`,
|
|
),
|
|
).toBe(false);
|
|
});
|
|
|
|
it('flags English progress text but not Korean or command-only text', () => {
|
|
expect(isEnglishProgress('Uncheck the import option, then continue.')).toBe(
|
|
true,
|
|
);
|
|
expect(isEnglishProgress('가져오기 옵션을 끄고 계속 진행하겠습니다.')).toBe(
|
|
false,
|
|
);
|
|
expect(isEnglishProgress('`bun run build` 실행 중입니다.')).toBe(false);
|
|
expect(isEnglishProgress('Done.')).toBe(false);
|
|
});
|
|
|
|
it('is only enabled for a Korean (or unset) AGENT_LANGUAGE', () => {
|
|
expect(isKoreanReplyTarget(undefined)).toBe(true);
|
|
expect(isKoreanReplyTarget('')).toBe(true);
|
|
expect(isKoreanReplyTarget('Korean')).toBe(true);
|
|
expect(isKoreanReplyTarget('ko')).toBe(true);
|
|
expect(isKoreanReplyTarget('English')).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe('Claude runner Korean rewrite', () => {
|
|
beforeEach(() => {
|
|
outputs.length = 0;
|
|
promptsSeen.length = 0;
|
|
delete process.env.AGENT_LANGUAGE;
|
|
});
|
|
afterEach(() => {
|
|
scriptedTurns = [];
|
|
});
|
|
|
|
it('asks once for a Korean rewrite and emits only the rewrite', async () => {
|
|
scriptedTurns = [
|
|
[assistant(ENGLISH_FINAL), result(ENGLISH_FINAL)],
|
|
[assistant(KOREAN_FINAL), result(KOREAN_FINAL)],
|
|
];
|
|
await run();
|
|
expect(promptsSeen).toEqual(['사용자 요청', KOREAN_REWRITE_NUDGE]);
|
|
expect(visible()).toEqual([['final', KOREAN_FINAL]]);
|
|
});
|
|
|
|
it('also catches the end_turn shortcut and ignores the stale result', async () => {
|
|
scriptedTurns = [
|
|
[assistant(ENGLISH_FINAL, true), result(ENGLISH_FINAL)],
|
|
[assistant(KOREAN_FINAL, true), result(KOREAN_FINAL)],
|
|
];
|
|
await run();
|
|
expect(promptsSeen).toEqual(['사용자 요청', KOREAN_REWRITE_NUDGE]);
|
|
expect(visible()).toEqual([['final', KOREAN_FINAL]]);
|
|
});
|
|
|
|
it('does not loop when the rewrite is still English', async () => {
|
|
const stillEnglish = `${ENGLISH_FINAL} (unchanged)`;
|
|
scriptedTurns = [
|
|
[assistant(ENGLISH_FINAL), result(ENGLISH_FINAL)],
|
|
[assistant(stillEnglish), result(stillEnglish)],
|
|
];
|
|
await run();
|
|
expect(promptsSeen).toHaveLength(2);
|
|
expect(visible()).toEqual([['final', stillEnglish]]);
|
|
});
|
|
|
|
it('falls back to the original answer when the rewrite errors', async () => {
|
|
scriptedTurns = [
|
|
[assistant(ENGLISH_FINAL), result(ENGLISH_FINAL)],
|
|
[result('', 'error_during_execution')],
|
|
];
|
|
await run();
|
|
expect(visible()).toEqual([
|
|
['final', `${ENGLISH_FINAL}${KOREAN_REWRITE_FAILED_NOTICE}`],
|
|
]);
|
|
});
|
|
|
|
it('falls back to the original answer when the stream ends early', async () => {
|
|
scriptedTurns = [[assistant(ENGLISH_FINAL), result(ENGLISH_FINAL)]];
|
|
await run();
|
|
expect(visible()).toEqual([
|
|
['final', `${ENGLISH_FINAL}${KOREAN_REWRITE_FAILED_NOTICE}`],
|
|
]);
|
|
});
|
|
|
|
it('drops English progress text flushed by tool_progress or tool results', async () => {
|
|
const englishProgress = 'Uncheck the import option, then continue.';
|
|
const koreanProgress = '가져오기 옵션을 끄고 계속 진행하겠습니다.';
|
|
scriptedTurns = [
|
|
[
|
|
assistant(englishProgress),
|
|
{ type: 'tool_progress', tool_name: 'Bash', elapsed_time_seconds: 3 },
|
|
assistant(koreanProgress),
|
|
{ type: 'user', message: { content: [] } },
|
|
assistant('Clicked Continue on the Trimble sign-in page.'),
|
|
{ type: 'user', message: { content: [] } },
|
|
assistant(KOREAN_FINAL),
|
|
result(KOREAN_FINAL),
|
|
],
|
|
];
|
|
await run();
|
|
expect(visible()).toEqual([
|
|
['intermediate', koreanProgress],
|
|
['final', KOREAN_FINAL],
|
|
]);
|
|
});
|
|
|
|
it('leaves Korean answers alone and emits them once', async () => {
|
|
scriptedTurns = [[assistant(KOREAN_FINAL), result(KOREAN_FINAL)]];
|
|
await run();
|
|
expect(promptsSeen).toEqual(['사용자 요청']);
|
|
expect(visible()).toEqual([['final', KOREAN_FINAL]]);
|
|
});
|
|
|
|
it('is disabled when AGENT_LANGUAGE is another language', async () => {
|
|
process.env.AGENT_LANGUAGE = 'English';
|
|
scriptedTurns = [[assistant(ENGLISH_FINAL), result(ENGLISH_FINAL)]];
|
|
await run();
|
|
expect(promptsSeen).toEqual(['사용자 요청']);
|
|
expect(visible()).toEqual([['final', ENGLISH_FINAL]]);
|
|
});
|
|
});
|