Files
EJClaw/runners/agent-runner/test/reply-language-guard.test.ts
Codex fdfdd384bf fix: mark untranslated fallbacks and drop English progress text
- When the Korean rewrite errors, the stream ends before it arrives, or a
  final must go out without a result event, append a Korean notice to the
  English text instead of sending it bare (content is kept: dropping a
  NEEDS_CONTEXT question would be worse than English).
- A still-English rewrite is sent unchanged: the nudge tells the model to
  repeat only when the user explicitly asked for English.
- Intermediate progress text cannot be rewritten mid-turn, so English
  progress (>= 20 Latin letters, Hangul < 10%) is dropped before it is
  flushed by tool_progress / tool results / the result event.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-24 02:11:42 +09:00

252 lines
9.3 KiB
TypeScript

import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const outputs: Array<{
status: string;
result: string | null;
phase?: string;
}> = [];
let scriptedTurns: Array<Array<Record<string, unknown>>> = [];
const promptsSeen: string[] = [];
vi.mock('@anthropic-ai/claude-agent-sdk', () => ({
query: ({ prompt }: { prompt: AsyncIterable<unknown> }) =>
(async function* () {
yield { type: 'system', subtype: 'init', session_id: 'sess-1' };
for await (const message of prompt) {
const content = (message as { message: { content: unknown } }).message
.content;
promptsSeen.push(
typeof content === 'string' ? content : JSON.stringify(content),
);
const turn = scriptedTurns.shift();
if (!turn) return; // CLI exits without answering
for (const event of turn) yield event;
}
})(),
}));
vi.mock('../src/claude-cli.js', () => ({
getClaudeCliPath: () => '/fake/claude',
}));
vi.mock('../src/output-protocol.js', async (importOriginal) => {
const actual =
await importOriginal<typeof import('../src/output-protocol.js')>();
return {
...actual,
writeOutput: (output: {
status: string;
result: string | null;
phase?: string;
}) => {
outputs.push({
status: output.status,
result: output.result,
phase: output.phase,
});
},
};
});
import { runClaudeQuery } from '../src/claude-query-runner.js';
import {
KOREAN_REWRITE_FAILED_NOTICE,
KOREAN_REWRITE_NUDGE,
isEnglishProgress,
isKoreanReplyTarget,
needsKoreanRewrite,
} from '../src/reply-language-guard.js';
const ENGLISH_FINAL =
'NEEDS_CONTEXT\n\nI accepted the Terms and entered the email and password, but Trimble now wants a verification code it emailed to you, so sign-in isn\'t finished yet.\n\n- I clicked "Accept" on the Trimble Offering Terms.\n- On the Trimble ID page, I rejected the cookie banner and entered your email.\n\n사용자 액션 아이템:\n1. Please send me the verification code.';
const KOREAN_FINAL =
'NEEDS_CONTEXT\n\n약관에 동의하고 이메일과 비밀번호를 입력했지만, Trimble이 이메일로 보낸 인증 코드를 요구해서 아직 로그인이 끝나지 않았습니다.';
// Real SDK assistant messages carry stop_reason only inside `message` (null
// while streaming); `topLevelEndTurn` exercises the runner's end_turn shortcut.
function assistant(text: string, topLevelEndTurn = false) {
return {
type: 'assistant',
...(topLevelEndTurn ? { stop_reason: 'end_turn' } : {}),
message: { content: [{ type: 'text', text }], stop_reason: null },
};
}
function result(text: string, subtype = 'success') {
return { type: 'result', subtype, result: text };
}
async function run() {
return runClaudeQuery({
prompt: '사용자 요청',
sessionId: undefined,
mcpServerPath: '/fake/ipc-mcp-stdio.js',
runnerInput: { prompt: '사용자 요청', groupFolder: 'g', chatJid: 'dc:1' },
sdkEnv: {},
reviewerRuntime: false,
claudeReadonlyReviewerRuntime: false,
claudeReadonlySandboxMode: null,
abortController: new AbortController(),
paths: {
groupDir: '/tmp/ejclaw-lang-guard-test',
groupFolder: 'g',
hostTasksDir: '/tmp/ejclaw-lang-guard-test/tasks',
ipcInputCloseSentinel: '/tmp/ejclaw-lang-guard-test/_close',
ipcInputDir: '/tmp/ejclaw-lang-guard-test/input',
workDir: '',
},
log: () => {},
});
}
// Text the host can post: finals and intermediate progress text (tool-progress
// timer labels like "Bash (3s)" are not prose and are left out).
const visible = () =>
outputs
.filter((o) => o.phase !== 'progress')
.map((o) => [o.phase ?? 'final', o.result]);
describe('needsKoreanRewrite', () => {
it('flags an English final answer', () => {
expect(needsKoreanRewrite(ENGLISH_FINAL)).toBe(true);
});
it('does not let Korean boilerplate lines hide an English answer', () => {
const english =
'TASK_DONE\n\nRelease 2.4.0 is out. The importer now keeps section planes, scene tabs and layer visibility when opening older files. Shadows render faster on machines without a graphics card, and the measure tool snaps to arc centers.\n\n사용자 액션 아이템: 없음\n리뷰 불필요: 요청받은 문구를 그대로 전달했을 뿐 변경 사항이 없습니다.';
expect(needsKoreanRewrite(english)).toBe(true);
});
it('keeps Korean answers full of identifiers, code and links', () => {
const korean =
'TASK_DONE\n\nv0.3.5 배포 완료했습니다. `src/installer-rp/pack.ts`의 `electron-builder` 설정을 고쳤습니다.\n\n```bash\nbun run build && git push origin main --follow-tags\n```\n릴리즈: https://git.tkrmagid.kr/tkrmagid/minecraft-resourcepack/releases/tag/v0.3.5\n- origin/main = 9efd4a6, CI green, assets uploaded';
expect(needsKoreanRewrite(korean)).toBe(false);
});
it('ignores short text, JSON and host error banners', () => {
expect(needsKoreanRewrite('TASK_DONE\n\nDone.')).toBe(false);
expect(needsKoreanRewrite(`{"answer": "${'x'.repeat(300)}"}`)).toBe(false);
expect(
needsKoreanRewrite(
`API Error: Opus 5.5's safeguards flagged this message. ${'word '.repeat(60)}`,
),
).toBe(false);
});
it('flags English progress text but not Korean or command-only text', () => {
expect(isEnglishProgress('Uncheck the import option, then continue.')).toBe(
true,
);
expect(isEnglishProgress('가져오기 옵션을 끄고 계속 진행하겠습니다.')).toBe(
false,
);
expect(isEnglishProgress('`bun run build` 실행 중입니다.')).toBe(false);
expect(isEnglishProgress('Done.')).toBe(false);
});
it('is only enabled for a Korean (or unset) AGENT_LANGUAGE', () => {
expect(isKoreanReplyTarget(undefined)).toBe(true);
expect(isKoreanReplyTarget('')).toBe(true);
expect(isKoreanReplyTarget('Korean')).toBe(true);
expect(isKoreanReplyTarget('ko')).toBe(true);
expect(isKoreanReplyTarget('English')).toBe(false);
});
});
describe('Claude runner Korean rewrite', () => {
beforeEach(() => {
outputs.length = 0;
promptsSeen.length = 0;
delete process.env.AGENT_LANGUAGE;
});
afterEach(() => {
scriptedTurns = [];
});
it('asks once for a Korean rewrite and emits only the rewrite', async () => {
scriptedTurns = [
[assistant(ENGLISH_FINAL), result(ENGLISH_FINAL)],
[assistant(KOREAN_FINAL), result(KOREAN_FINAL)],
];
await run();
expect(promptsSeen).toEqual(['사용자 요청', KOREAN_REWRITE_NUDGE]);
expect(visible()).toEqual([['final', KOREAN_FINAL]]);
});
it('also catches the end_turn shortcut and ignores the stale result', async () => {
scriptedTurns = [
[assistant(ENGLISH_FINAL, true), result(ENGLISH_FINAL)],
[assistant(KOREAN_FINAL, true), result(KOREAN_FINAL)],
];
await run();
expect(promptsSeen).toEqual(['사용자 요청', KOREAN_REWRITE_NUDGE]);
expect(visible()).toEqual([['final', KOREAN_FINAL]]);
});
it('does not loop when the rewrite is still English', async () => {
const stillEnglish = `${ENGLISH_FINAL} (unchanged)`;
scriptedTurns = [
[assistant(ENGLISH_FINAL), result(ENGLISH_FINAL)],
[assistant(stillEnglish), result(stillEnglish)],
];
await run();
expect(promptsSeen).toHaveLength(2);
expect(visible()).toEqual([['final', stillEnglish]]);
});
it('falls back to the original answer when the rewrite errors', async () => {
scriptedTurns = [
[assistant(ENGLISH_FINAL), result(ENGLISH_FINAL)],
[result('', 'error_during_execution')],
];
await run();
expect(visible()).toEqual([
['final', `${ENGLISH_FINAL}${KOREAN_REWRITE_FAILED_NOTICE}`],
]);
});
it('falls back to the original answer when the stream ends early', async () => {
scriptedTurns = [[assistant(ENGLISH_FINAL), result(ENGLISH_FINAL)]];
await run();
expect(visible()).toEqual([
['final', `${ENGLISH_FINAL}${KOREAN_REWRITE_FAILED_NOTICE}`],
]);
});
it('drops English progress text flushed by tool_progress or tool results', async () => {
const englishProgress = 'Uncheck the import option, then continue.';
const koreanProgress = '가져오기 옵션을 끄고 계속 진행하겠습니다.';
scriptedTurns = [
[
assistant(englishProgress),
{ type: 'tool_progress', tool_name: 'Bash', elapsed_time_seconds: 3 },
assistant(koreanProgress),
{ type: 'user', message: { content: [] } },
assistant('Clicked Continue on the Trimble sign-in page.'),
{ type: 'user', message: { content: [] } },
assistant(KOREAN_FINAL),
result(KOREAN_FINAL),
],
];
await run();
expect(visible()).toEqual([
['intermediate', koreanProgress],
['final', KOREAN_FINAL],
]);
});
it('leaves Korean answers alone and emits them once', async () => {
scriptedTurns = [[assistant(KOREAN_FINAL), result(KOREAN_FINAL)]];
await run();
expect(promptsSeen).toEqual(['사용자 요청']);
expect(visible()).toEqual([['final', KOREAN_FINAL]]);
});
it('is disabled when AGENT_LANGUAGE is another language', async () => {
process.env.AGENT_LANGUAGE = 'English';
scriptedTurns = [[assistant(ENGLISH_FINAL), result(ENGLISH_FINAL)]];
await run();
expect(promptsSeen).toEqual(['사용자 요청']);
expect(visible()).toEqual([['final', ENGLISH_FINAL]]);
});
});