fix: mark untranslated fallbacks and drop English progress text
- When the Korean rewrite errors, the stream ends before it arrives, or a final must go out without a result event, append a Korean notice to the English text instead of sending it bare (content is kept: dropping a NEEDS_CONTEXT question would be worse than English). - A still-English rewrite is sent unchanged: the nudge tells the model to repeat only when the user explicitly asked for English. - Intermediate progress text cannot be rewritten mid-turn, so English progress (>= 20 Latin letters, Hangul < 10%) is dropped before it is flushed by tool_progress / tool results / the result event. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -25,7 +25,9 @@ import {
|
||||
writeOutput,
|
||||
} from './output-protocol.js';
|
||||
import {
|
||||
KOREAN_REWRITE_FAILED_NOTICE,
|
||||
KOREAN_REWRITE_NUDGE,
|
||||
isEnglishProgress,
|
||||
isKoreanReplyTarget,
|
||||
needsKoreanRewrite,
|
||||
} from './reply-language-guard.js';
|
||||
@@ -180,7 +182,9 @@ class ClaudeQueryRunner {
|
||||
);
|
||||
writeOutput({
|
||||
status: 'success',
|
||||
...normalizeStructuredOutput(this.pendingProgressText),
|
||||
...normalizeStructuredOutput(
|
||||
this.withLanguageNotice(this.pendingProgressText),
|
||||
),
|
||||
newSessionId: this.newSessionId,
|
||||
toolUsage: this.toolUsage.snapshot(),
|
||||
});
|
||||
@@ -530,12 +534,7 @@ class ClaudeQueryRunner {
|
||||
);
|
||||
textResult = this.pendingProgressText;
|
||||
} else {
|
||||
writeOutput({
|
||||
status: 'success',
|
||||
phase: 'intermediate',
|
||||
result: this.pendingProgressText,
|
||||
newSessionId: this.newSessionId,
|
||||
});
|
||||
this.flushPendingProgressAsIntermediate();
|
||||
}
|
||||
this.pendingProgressText = null;
|
||||
}
|
||||
@@ -545,7 +544,9 @@ class ClaudeQueryRunner {
|
||||
);
|
||||
if (isError && this.languageRewrite === 'pending') {
|
||||
this.args.log('Korean rewrite failed, emitting the original answer');
|
||||
this.writeSuccessResult(this.languageRewriteSource);
|
||||
this.writeSuccessResult(
|
||||
`${this.languageRewriteSource ?? ''}${KOREAN_REWRITE_FAILED_NOTICE}`,
|
||||
);
|
||||
this.languageRewrite = 'done';
|
||||
} else if (isError) {
|
||||
this.writeErrorResult(resultMessage, textResult);
|
||||
@@ -670,13 +671,24 @@ class ClaudeQueryRunner {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Final text that must go out without a result event (stream closed), so it
|
||||
* cannot be rewritten: mark it if it is English.
|
||||
*/
|
||||
private withLanguageNotice(text: string): string {
|
||||
if (!this.languageGuardEnabled || !needsKoreanRewrite(text)) return text;
|
||||
return `${text}${KOREAN_REWRITE_FAILED_NOTICE}`;
|
||||
}
|
||||
|
||||
/** Emit the original answer if the stream ends before the rewrite arrives. */
|
||||
private flushLanguageRewriteFallback(): void {
|
||||
if (this.languageRewrite !== 'pending' || this.terminalResultObserved) {
|
||||
return;
|
||||
}
|
||||
this.args.log('Korean rewrite not received, emitting the original answer');
|
||||
this.writeSuccessResult(this.languageRewriteSource);
|
||||
this.writeSuccessResult(
|
||||
`${this.languageRewriteSource ?? ''}${KOREAN_REWRITE_FAILED_NOTICE}`,
|
||||
);
|
||||
this.languageRewrite = 'done';
|
||||
this.terminalResultObserved = true;
|
||||
this.resultCount++;
|
||||
@@ -684,6 +696,16 @@ class ClaudeQueryRunner {
|
||||
|
||||
private flushPendingProgressAsIntermediate(): void {
|
||||
if (!this.pendingProgressText) return;
|
||||
if (
|
||||
this.languageGuardEnabled &&
|
||||
isEnglishProgress(this.pendingProgressText)
|
||||
) {
|
||||
this.args.log(
|
||||
`Dropping English progress text (${this.pendingProgressText.length} chars)`,
|
||||
);
|
||||
this.pendingProgressText = null;
|
||||
return;
|
||||
}
|
||||
writeOutput({
|
||||
status: 'success',
|
||||
phase: 'intermediate',
|
||||
@@ -701,7 +723,9 @@ class ClaudeQueryRunner {
|
||||
);
|
||||
writeOutput({
|
||||
status: 'success',
|
||||
...normalizeStructuredOutput(this.pendingProgressText),
|
||||
...normalizeStructuredOutput(
|
||||
this.withLanguageNotice(this.pendingProgressText),
|
||||
),
|
||||
newSessionId: this.newSessionId,
|
||||
...buildCompactionOutput(this.compaction),
|
||||
toolUsage: this.toolUsage.snapshot(),
|
||||
|
||||
@@ -23,6 +23,14 @@ export const KOREAN_REWRITE_NUDGE =
|
||||
'내용은 바꾸거나 줄이지 말고, 첫 줄의 상태 키워드(TASK_DONE, STEP_DONE, NEEDS_CONTEXT, REVISE 등), 코드, 명령어, 파일 경로, 링크, MEDIA: 줄, 화면에 보이는 영문 버튼·메뉴 이름은 그대로 두세요. ' +
|
||||
'도구는 쓰지 마세요. 사용자가 영어를 명시적으로 요청했다면 방금 답변을 그대로 다시 출력하세요.';
|
||||
|
||||
// Appended (never prepended: the first line carries the status keyword) when
|
||||
// the rewrite could not be produced and the English original is sent instead.
|
||||
export const KOREAN_REWRITE_FAILED_NOTICE =
|
||||
'\n\n[자동 안내] 이 답변을 한국어로 다시 쓰지 못해 원문을 그대로 보냅니다. 한국어로 다시 받으려면 "한국어로 다시 알려주세요"라고 보내 주세요.';
|
||||
|
||||
// Progress text is short, so it gets a lower floor than final answers.
|
||||
const MIN_PROGRESS_LATIN_LETTERS = 20;
|
||||
|
||||
function stripNonProse(text: string): string {
|
||||
return (
|
||||
text
|
||||
@@ -45,6 +53,25 @@ export function isKoreanReplyTarget(
|
||||
return /^(ko|ko-kr|korean|한국어)$/i.test(target);
|
||||
}
|
||||
|
||||
function isMostlyLatin(text: string, minLatin: number): boolean {
|
||||
const prose = stripNonProse(text);
|
||||
const latin = prose.match(/[A-Za-z]/g)?.length ?? 0;
|
||||
if (latin < minLatin) return false;
|
||||
const hangul = prose.match(/[\uac00-\ud7a3]/g)?.length ?? 0;
|
||||
return hangul < latin * MAX_HANGUL_TO_LATIN_RATIO;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when intermediate progress text is English. Progress cannot be
|
||||
* rewritten mid-turn, so the runner drops it instead of posting it.
|
||||
*/
|
||||
export function isEnglishProgress(text: string | null | undefined): boolean {
|
||||
if (!text) return false;
|
||||
const trimmed = text.trim();
|
||||
if (trimmed.startsWith('{')) return false;
|
||||
return isMostlyLatin(trimmed, MIN_PROGRESS_LATIN_LETTERS);
|
||||
}
|
||||
|
||||
/** True when `text` is a final answer written in English instead of Korean. */
|
||||
export function needsKoreanRewrite(text: string | null | undefined): boolean {
|
||||
if (!text) return false;
|
||||
@@ -53,9 +80,5 @@ export function needsKoreanRewrite(text: string | null | undefined): boolean {
|
||||
if (HOST_ERROR_PREFIXES.some((prefix) => trimmed.startsWith(prefix))) {
|
||||
return false;
|
||||
}
|
||||
const prose = stripNonProse(trimmed);
|
||||
const latin = prose.match(/[A-Za-z]/g)?.length ?? 0;
|
||||
if (latin < MIN_LATIN_LETTERS) return false;
|
||||
const hangul = prose.match(/[가-힣]/g)?.length ?? 0;
|
||||
return hangul < latin * MAX_HANGUL_TO_LATIN_RATIO;
|
||||
return isMostlyLatin(trimmed, MIN_LATIN_LETTERS);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const outputs: Array<{ status: string; result: string | null }> = [];
|
||||
const outputs: Array<{
|
||||
status: string;
|
||||
result: string | null;
|
||||
phase?: string;
|
||||
}> = [];
|
||||
let scriptedTurns: Array<Array<Record<string, unknown>>> = [];
|
||||
const promptsSeen: string[] = [];
|
||||
|
||||
@@ -30,15 +34,25 @@ vi.mock('../src/output-protocol.js', async (importOriginal) => {
|
||||
await importOriginal<typeof import('../src/output-protocol.js')>();
|
||||
return {
|
||||
...actual,
|
||||
writeOutput: (output: { status: string; result: string | null }) => {
|
||||
outputs.push({ status: output.status, result: output.result });
|
||||
writeOutput: (output: {
|
||||
status: string;
|
||||
result: string | null;
|
||||
phase?: string;
|
||||
}) => {
|
||||
outputs.push({
|
||||
status: output.status,
|
||||
result: output.result,
|
||||
phase: output.phase,
|
||||
});
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
import { runClaudeQuery } from '../src/claude-query-runner.js';
|
||||
import {
|
||||
KOREAN_REWRITE_FAILED_NOTICE,
|
||||
KOREAN_REWRITE_NUDGE,
|
||||
isEnglishProgress,
|
||||
isKoreanReplyTarget,
|
||||
needsKoreanRewrite,
|
||||
} from '../src/reply-language-guard.js';
|
||||
@@ -84,7 +98,12 @@ async function run() {
|
||||
});
|
||||
}
|
||||
|
||||
const visible = () => outputs.map((o) => [o.status, o.result]);
|
||||
// Text the host can post: finals and intermediate progress text (tool-progress
|
||||
// timer labels like "Bash (3s)" are not prose and are left out).
|
||||
const visible = () =>
|
||||
outputs
|
||||
.filter((o) => o.phase !== 'progress')
|
||||
.map((o) => [o.phase ?? 'final', o.result]);
|
||||
|
||||
describe('needsKoreanRewrite', () => {
|
||||
it('flags an English final answer', () => {
|
||||
@@ -113,6 +132,17 @@ describe('needsKoreanRewrite', () => {
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('flags English progress text but not Korean or command-only text', () => {
|
||||
expect(isEnglishProgress('Uncheck the import option, then continue.')).toBe(
|
||||
true,
|
||||
);
|
||||
expect(isEnglishProgress('가져오기 옵션을 끄고 계속 진행하겠습니다.')).toBe(
|
||||
false,
|
||||
);
|
||||
expect(isEnglishProgress('`bun run build` 실행 중입니다.')).toBe(false);
|
||||
expect(isEnglishProgress('Done.')).toBe(false);
|
||||
});
|
||||
|
||||
it('is only enabled for a Korean (or unset) AGENT_LANGUAGE', () => {
|
||||
expect(isKoreanReplyTarget(undefined)).toBe(true);
|
||||
expect(isKoreanReplyTarget('')).toBe(true);
|
||||
@@ -139,7 +169,7 @@ describe('Claude runner Korean rewrite', () => {
|
||||
];
|
||||
await run();
|
||||
expect(promptsSeen).toEqual(['사용자 요청', KOREAN_REWRITE_NUDGE]);
|
||||
expect(visible()).toEqual([['success', KOREAN_FINAL]]);
|
||||
expect(visible()).toEqual([['final', KOREAN_FINAL]]);
|
||||
});
|
||||
|
||||
it('also catches the end_turn shortcut and ignores the stale result', async () => {
|
||||
@@ -149,7 +179,7 @@ describe('Claude runner Korean rewrite', () => {
|
||||
];
|
||||
await run();
|
||||
expect(promptsSeen).toEqual(['사용자 요청', KOREAN_REWRITE_NUDGE]);
|
||||
expect(visible()).toEqual([['success', KOREAN_FINAL]]);
|
||||
expect(visible()).toEqual([['final', KOREAN_FINAL]]);
|
||||
});
|
||||
|
||||
it('does not loop when the rewrite is still English', async () => {
|
||||
@@ -160,7 +190,7 @@ describe('Claude runner Korean rewrite', () => {
|
||||
];
|
||||
await run();
|
||||
expect(promptsSeen).toHaveLength(2);
|
||||
expect(visible()).toEqual([['success', stillEnglish]]);
|
||||
expect(visible()).toEqual([['final', stillEnglish]]);
|
||||
});
|
||||
|
||||
it('falls back to the original answer when the rewrite errors', async () => {
|
||||
@@ -169,20 +199,46 @@ describe('Claude runner Korean rewrite', () => {
|
||||
[result('', 'error_during_execution')],
|
||||
];
|
||||
await run();
|
||||
expect(visible()).toEqual([['success', ENGLISH_FINAL]]);
|
||||
expect(visible()).toEqual([
|
||||
['final', `${ENGLISH_FINAL}${KOREAN_REWRITE_FAILED_NOTICE}`],
|
||||
]);
|
||||
});
|
||||
|
||||
it('falls back to the original answer when the stream ends early', async () => {
|
||||
scriptedTurns = [[assistant(ENGLISH_FINAL), result(ENGLISH_FINAL)]];
|
||||
await run();
|
||||
expect(visible()).toEqual([['success', ENGLISH_FINAL]]);
|
||||
expect(visible()).toEqual([
|
||||
['final', `${ENGLISH_FINAL}${KOREAN_REWRITE_FAILED_NOTICE}`],
|
||||
]);
|
||||
});
|
||||
|
||||
it('drops English progress text flushed by tool_progress or tool results', async () => {
|
||||
const englishProgress = 'Uncheck the import option, then continue.';
|
||||
const koreanProgress = '가져오기 옵션을 끄고 계속 진행하겠습니다.';
|
||||
scriptedTurns = [
|
||||
[
|
||||
assistant(englishProgress),
|
||||
{ type: 'tool_progress', tool_name: 'Bash', elapsed_time_seconds: 3 },
|
||||
assistant(koreanProgress),
|
||||
{ type: 'user', message: { content: [] } },
|
||||
assistant('Clicked Continue on the Trimble sign-in page.'),
|
||||
{ type: 'user', message: { content: [] } },
|
||||
assistant(KOREAN_FINAL),
|
||||
result(KOREAN_FINAL),
|
||||
],
|
||||
];
|
||||
await run();
|
||||
expect(visible()).toEqual([
|
||||
['intermediate', koreanProgress],
|
||||
['final', KOREAN_FINAL],
|
||||
]);
|
||||
});
|
||||
|
||||
it('leaves Korean answers alone and emits them once', async () => {
|
||||
scriptedTurns = [[assistant(KOREAN_FINAL), result(KOREAN_FINAL)]];
|
||||
await run();
|
||||
expect(promptsSeen).toEqual(['사용자 요청']);
|
||||
expect(visible()).toEqual([['success', KOREAN_FINAL]]);
|
||||
expect(visible()).toEqual([['final', KOREAN_FINAL]]);
|
||||
});
|
||||
|
||||
it('is disabled when AGENT_LANGUAGE is another language', async () => {
|
||||
@@ -190,6 +246,6 @@ describe('Claude runner Korean rewrite', () => {
|
||||
scriptedTurns = [[assistant(ENGLISH_FINAL), result(ENGLISH_FINAL)]];
|
||||
await run();
|
||||
expect(promptsSeen).toEqual(['사용자 요청']);
|
||||
expect(visible()).toEqual([['success', ENGLISH_FINAL]]);
|
||||
expect(visible()).toEqual([['final', ENGLISH_FINAL]]);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user