fix: mark untranslated fallbacks and drop English progress text

- When the Korean rewrite errors, the stream ends before it arrives, or a
  final must go out without a result event, append a Korean notice to the
  English text instead of sending it bare (content is kept: dropping a
  NEEDS_CONTEXT question would be worse than English).
- A still-English rewrite is sent unchanged: the nudge tells the model to
  repeat only when the user explicitly asked for English.
- Intermediate progress text cannot be rewritten mid-turn, so English
  progress (>= 20 Latin letters, Hangul < 10%) is dropped before it is
  flushed by tool_progress / tool results / the result event.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Codex
2026-10-24 02:11:42 +09:00
parent 98011e282b
commit fdfdd384bf
3 changed files with 129 additions and 26 deletions

View File

@@ -25,7 +25,9 @@ import {
writeOutput,
} from './output-protocol.js';
import {
KOREAN_REWRITE_FAILED_NOTICE,
KOREAN_REWRITE_NUDGE,
isEnglishProgress,
isKoreanReplyTarget,
needsKoreanRewrite,
} from './reply-language-guard.js';
@@ -180,7 +182,9 @@ class ClaudeQueryRunner {
);
writeOutput({
status: 'success',
...normalizeStructuredOutput(this.pendingProgressText),
...normalizeStructuredOutput(
this.withLanguageNotice(this.pendingProgressText),
),
newSessionId: this.newSessionId,
toolUsage: this.toolUsage.snapshot(),
});
@@ -530,12 +534,7 @@ class ClaudeQueryRunner {
);
textResult = this.pendingProgressText;
} else {
writeOutput({
status: 'success',
phase: 'intermediate',
result: this.pendingProgressText,
newSessionId: this.newSessionId,
});
this.flushPendingProgressAsIntermediate();
}
this.pendingProgressText = null;
}
@@ -545,7 +544,9 @@ class ClaudeQueryRunner {
);
if (isError && this.languageRewrite === 'pending') {
this.args.log('Korean rewrite failed, emitting the original answer');
this.writeSuccessResult(this.languageRewriteSource);
this.writeSuccessResult(
`${this.languageRewriteSource ?? ''}${KOREAN_REWRITE_FAILED_NOTICE}`,
);
this.languageRewrite = 'done';
} else if (isError) {
this.writeErrorResult(resultMessage, textResult);
@@ -670,13 +671,24 @@ class ClaudeQueryRunner {
return true;
}
/**
* Final text that must go out without a result event (stream closed), so it
* cannot be rewritten: mark it if it is English.
*/
private withLanguageNotice(text: string): string {
if (!this.languageGuardEnabled || !needsKoreanRewrite(text)) return text;
return `${text}${KOREAN_REWRITE_FAILED_NOTICE}`;
}
/** Emit the original answer if the stream ends before the rewrite arrives. */
private flushLanguageRewriteFallback(): void {
if (this.languageRewrite !== 'pending' || this.terminalResultObserved) {
return;
}
this.args.log('Korean rewrite not received, emitting the original answer');
this.writeSuccessResult(this.languageRewriteSource);
this.writeSuccessResult(
`${this.languageRewriteSource ?? ''}${KOREAN_REWRITE_FAILED_NOTICE}`,
);
this.languageRewrite = 'done';
this.terminalResultObserved = true;
this.resultCount++;
@@ -684,6 +696,16 @@ class ClaudeQueryRunner {
private flushPendingProgressAsIntermediate(): void {
if (!this.pendingProgressText) return;
if (
this.languageGuardEnabled &&
isEnglishProgress(this.pendingProgressText)
) {
this.args.log(
`Dropping English progress text (${this.pendingProgressText.length} chars)`,
);
this.pendingProgressText = null;
return;
}
writeOutput({
status: 'success',
phase: 'intermediate',
@@ -701,7 +723,9 @@ class ClaudeQueryRunner {
);
writeOutput({
status: 'success',
...normalizeStructuredOutput(this.pendingProgressText),
...normalizeStructuredOutput(
this.withLanguageNotice(this.pendingProgressText),
),
newSessionId: this.newSessionId,
...buildCompactionOutput(this.compaction),
toolUsage: this.toolUsage.snapshot(),

View File

@@ -23,6 +23,14 @@ export const KOREAN_REWRITE_NUDGE =
'내용은 바꾸거나 줄이지 말고, 첫 줄의 상태 키워드(TASK_DONE, STEP_DONE, NEEDS_CONTEXT, REVISE 등), 코드, 명령어, 파일 경로, 링크, MEDIA: 줄, 화면에 보이는 영문 버튼·메뉴 이름은 그대로 두세요. ' +
'도구는 쓰지 마세요. 사용자가 영어를 명시적으로 요청했다면 방금 답변을 그대로 다시 출력하세요.';
// Appended (never prepended: the first line carries the status keyword) when
// the rewrite could not be produced and the English original is sent instead.
export const KOREAN_REWRITE_FAILED_NOTICE =
'\n\n[자동 안내] 이 답변을 한국어로 다시 쓰지 못해 원문을 그대로 보냅니다. 한국어로 다시 받으려면 "한국어로 다시 알려주세요"라고 보내 주세요.';
// Progress text is short, so it gets a lower floor than final answers.
const MIN_PROGRESS_LATIN_LETTERS = 20;
function stripNonProse(text: string): string {
return (
text
@@ -45,6 +53,25 @@ export function isKoreanReplyTarget(
return /^(ko|ko-kr|korean|한국어)$/i.test(target);
}
function isMostlyLatin(text: string, minLatin: number): boolean {
const prose = stripNonProse(text);
const latin = prose.match(/[A-Za-z]/g)?.length ?? 0;
if (latin < minLatin) return false;
const hangul = prose.match(/[\uac00-\ud7a3]/g)?.length ?? 0;
return hangul < latin * MAX_HANGUL_TO_LATIN_RATIO;
}
/**
* True when intermediate progress text is English. Progress cannot be
* rewritten mid-turn, so the runner drops it instead of posting it.
*/
export function isEnglishProgress(text: string | null | undefined): boolean {
if (!text) return false;
const trimmed = text.trim();
if (trimmed.startsWith('{')) return false;
return isMostlyLatin(trimmed, MIN_PROGRESS_LATIN_LETTERS);
}
/** True when `text` is a final answer written in English instead of Korean. */
export function needsKoreanRewrite(text: string | null | undefined): boolean {
if (!text) return false;
@@ -53,9 +80,5 @@ export function needsKoreanRewrite(text: string | null | undefined): boolean {
if (HOST_ERROR_PREFIXES.some((prefix) => trimmed.startsWith(prefix))) {
return false;
}
const prose = stripNonProse(trimmed);
const latin = prose.match(/[A-Za-z]/g)?.length ?? 0;
if (latin < MIN_LATIN_LETTERS) return false;
const hangul = prose.match(/[가-힣]/g)?.length ?? 0;
return hangul < latin * MAX_HANGUL_TO_LATIN_RATIO;
return isMostlyLatin(trimmed, MIN_LATIN_LETTERS);
}

View File

@@ -1,6 +1,10 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const outputs: Array<{ status: string; result: string | null }> = [];
const outputs: Array<{
status: string;
result: string | null;
phase?: string;
}> = [];
let scriptedTurns: Array<Array<Record<string, unknown>>> = [];
const promptsSeen: string[] = [];
@@ -30,15 +34,25 @@ vi.mock('../src/output-protocol.js', async (importOriginal) => {
await importOriginal<typeof import('../src/output-protocol.js')>();
return {
...actual,
writeOutput: (output: { status: string; result: string | null }) => {
outputs.push({ status: output.status, result: output.result });
writeOutput: (output: {
status: string;
result: string | null;
phase?: string;
}) => {
outputs.push({
status: output.status,
result: output.result,
phase: output.phase,
});
},
};
});
import { runClaudeQuery } from '../src/claude-query-runner.js';
import {
KOREAN_REWRITE_FAILED_NOTICE,
KOREAN_REWRITE_NUDGE,
isEnglishProgress,
isKoreanReplyTarget,
needsKoreanRewrite,
} from '../src/reply-language-guard.js';
@@ -84,7 +98,12 @@ async function run() {
});
}
const visible = () => outputs.map((o) => [o.status, o.result]);
// Text the host can post: finals and intermediate progress text (tool-progress
// timer labels like "Bash (3s)" are not prose and are left out).
const visible = () =>
outputs
.filter((o) => o.phase !== 'progress')
.map((o) => [o.phase ?? 'final', o.result]);
describe('needsKoreanRewrite', () => {
it('flags an English final answer', () => {
@@ -113,6 +132,17 @@ describe('needsKoreanRewrite', () => {
).toBe(false);
});
it('flags English progress text but not Korean or command-only text', () => {
expect(isEnglishProgress('Uncheck the import option, then continue.')).toBe(
true,
);
expect(isEnglishProgress('가져오기 옵션을 끄고 계속 진행하겠습니다.')).toBe(
false,
);
expect(isEnglishProgress('`bun run build` 실행 중입니다.')).toBe(false);
expect(isEnglishProgress('Done.')).toBe(false);
});
it('is only enabled for a Korean (or unset) AGENT_LANGUAGE', () => {
expect(isKoreanReplyTarget(undefined)).toBe(true);
expect(isKoreanReplyTarget('')).toBe(true);
@@ -139,7 +169,7 @@ describe('Claude runner Korean rewrite', () => {
];
await run();
expect(promptsSeen).toEqual(['사용자 요청', KOREAN_REWRITE_NUDGE]);
expect(visible()).toEqual([['success', KOREAN_FINAL]]);
expect(visible()).toEqual([['final', KOREAN_FINAL]]);
});
it('also catches the end_turn shortcut and ignores the stale result', async () => {
@@ -149,7 +179,7 @@ describe('Claude runner Korean rewrite', () => {
];
await run();
expect(promptsSeen).toEqual(['사용자 요청', KOREAN_REWRITE_NUDGE]);
expect(visible()).toEqual([['success', KOREAN_FINAL]]);
expect(visible()).toEqual([['final', KOREAN_FINAL]]);
});
it('does not loop when the rewrite is still English', async () => {
@@ -160,7 +190,7 @@ describe('Claude runner Korean rewrite', () => {
];
await run();
expect(promptsSeen).toHaveLength(2);
expect(visible()).toEqual([['success', stillEnglish]]);
expect(visible()).toEqual([['final', stillEnglish]]);
});
it('falls back to the original answer when the rewrite errors', async () => {
@@ -169,20 +199,46 @@ describe('Claude runner Korean rewrite', () => {
[result('', 'error_during_execution')],
];
await run();
expect(visible()).toEqual([['success', ENGLISH_FINAL]]);
expect(visible()).toEqual([
['final', `${ENGLISH_FINAL}${KOREAN_REWRITE_FAILED_NOTICE}`],
]);
});
it('falls back to the original answer when the stream ends early', async () => {
scriptedTurns = [[assistant(ENGLISH_FINAL), result(ENGLISH_FINAL)]];
await run();
expect(visible()).toEqual([['success', ENGLISH_FINAL]]);
expect(visible()).toEqual([
['final', `${ENGLISH_FINAL}${KOREAN_REWRITE_FAILED_NOTICE}`],
]);
});
it('drops English progress text flushed by tool_progress or tool results', async () => {
const englishProgress = 'Uncheck the import option, then continue.';
const koreanProgress = '가져오기 옵션을 끄고 계속 진행하겠습니다.';
scriptedTurns = [
[
assistant(englishProgress),
{ type: 'tool_progress', tool_name: 'Bash', elapsed_time_seconds: 3 },
assistant(koreanProgress),
{ type: 'user', message: { content: [] } },
assistant('Clicked Continue on the Trimble sign-in page.'),
{ type: 'user', message: { content: [] } },
assistant(KOREAN_FINAL),
result(KOREAN_FINAL),
],
];
await run();
expect(visible()).toEqual([
['intermediate', koreanProgress],
['final', KOREAN_FINAL],
]);
});
it('leaves Korean answers alone and emits them once', async () => {
scriptedTurns = [[assistant(KOREAN_FINAL), result(KOREAN_FINAL)]];
await run();
expect(promptsSeen).toEqual(['사용자 요청']);
expect(visible()).toEqual([['success', KOREAN_FINAL]]);
expect(visible()).toEqual([['final', KOREAN_FINAL]]);
});
it('is disabled when AGENT_LANGUAGE is another language', async () => {
@@ -190,6 +246,6 @@ describe('Claude runner Korean rewrite', () => {
scriptedTurns = [[assistant(ENGLISH_FINAL), result(ENGLISH_FINAL)]];
await run();
expect(promptsSeen).toEqual(['사용자 요청']);
expect(visible()).toEqual([['success', ENGLISH_FINAL]]);
expect(visible()).toEqual([['final', ENGLISH_FINAL]]);
});
});