d5fd218c86
A 4b model replied to "하이" with "테니스 연습을 Trenches Gym에서..." — it copied the literal "box at Trenches Gym" few-shot example embedded in the persona prompt and mangled boxing into tennis, presenting a prompt example as if it were a real user fact. Remove the copyable proper-noun example and add an explicit guard: use ONLY names/places/activities that literally appear in the memory section, never borrow them from the instructions or example wording.
200 lines
8.9 KiB
Python
200 lines
8.9 KiB
Python
"""Tests for the unified persona system prompt.
|
|
|
|
The persona should match the user's configured wake word so renaming the
|
|
wake word to e.g. "Friday" produces a butler named Friday, not one still
|
|
hardcoded to Jarvis.
|
|
"""
|
|
|
|
from jarvis.system_prompt import (
|
|
build_system_prompt,
|
|
load_agent_instructions,
|
|
output_language_directive,
|
|
reply_language_directive,
|
|
ENGLISH_ONLY_DIRECTIVE,
|
|
)
|
|
|
|
|
|
class TestBuildSystemPrompt:
|
|
def test_default_name_is_jarvis(self):
|
|
prompt = build_system_prompt()
|
|
assert "named Jarvis" in prompt
|
|
|
|
def test_custom_name_replaces_jarvis(self):
|
|
prompt = build_system_prompt("Friday")
|
|
assert "named Friday" in prompt
|
|
assert "named Jarvis" not in prompt
|
|
|
|
def test_lowercase_wake_word_is_capitalised(self):
|
|
prompt = build_system_prompt("friday".capitalize())
|
|
assert "named Friday" in prompt
|
|
|
|
def test_blank_name_falls_back_to_jarvis(self):
|
|
assert "named Jarvis" in build_system_prompt("")
|
|
assert "named Jarvis" in build_system_prompt(" ")
|
|
assert "named Jarvis" in build_system_prompt(None) # type: ignore[arg-type]
|
|
|
|
def test_default_keeps_user_language_clause(self):
|
|
# Without a lock, the persona still mirrors the user's language.
|
|
assert "in the user's language" in build_system_prompt("Jarvis")
|
|
|
|
def test_language_lock_rewrites_user_language_clause(self):
|
|
# With a lock, the contradicting "user's language" clause is rewritten
|
|
# so the persona does not fight the OUTPUT_LANGUAGE directive.
|
|
prompt = build_system_prompt("Jarvis", "Korean")
|
|
assert "in the user's language" not in prompt
|
|
assert "in Korean" in prompt
|
|
|
|
def test_persona_enforces_single_sentence(self):
|
|
# Spoken replies must be one sentence (TTS latency scales with text
|
|
# length, and the user asked for one-sentence answers). The persona must
|
|
# state the single-sentence rule and must NOT carry the old "two at the
|
|
# very most" allowance that let the model run long.
|
|
prompt = build_system_prompt("Jarvis")
|
|
assert "single short sentence" in prompt
|
|
assert "Never write a second sentence" in prompt
|
|
assert "two at the very most" not in prompt
|
|
|
|
def test_persona_aside_does_not_authorise_a_second_sentence(self):
|
|
# The dry aside must fold into the one sentence, not become a 2nd one.
|
|
prompt = build_system_prompt("Jarvis")
|
|
assert "SINGLE sentence" in prompt
|
|
assert "never add it as " in prompt
|
|
|
|
def test_persona_has_no_copyable_proper_noun_examples(self):
|
|
# A weak model parroted the literal "Trenches Gym" example from the
|
|
# persona as if it were a real user fact (boxing mangled to tennis).
|
|
# The persona must not embed copyable personal proper nouns, and must
|
|
# tell the model to use ONLY facts that literally appear in the memory
|
|
# section — never borrow names/places from the instructions themselves.
|
|
prompt = build_system_prompt("Jarvis")
|
|
assert "Trenches" not in prompt
|
|
assert "never borrow any name, place, or activity from these instructions" in prompt
|
|
|
|
|
|
class TestOutputLanguageDirective:
|
|
"""A deployment may lock replies to a single language via OUTPUT_LANGUAGE.
|
|
|
|
Unset (the default) must keep the assistant's multilingual behaviour of
|
|
replying in the user's own language, so the helper returns None and no
|
|
directive is injected.
|
|
"""
|
|
|
|
def test_unset_returns_none(self):
|
|
assert output_language_directive(None) is None
|
|
assert output_language_directive("") is None
|
|
assert output_language_directive(" ") is None
|
|
|
|
def test_set_language_is_named_and_exclusive(self):
|
|
directive = output_language_directive("Korean")
|
|
assert directive is not None
|
|
assert "Korean" in directive
|
|
# Must force exclusivity, not merely prefer the language.
|
|
assert "only" in directive.lower()
|
|
|
|
def test_language_agnostic(self):
|
|
# The helper takes any language string — no hardcoded single language.
|
|
assert "French" in (output_language_directive("French") or "")
|
|
assert "日本語" in (output_language_directive("日本語") or "")
|
|
|
|
def test_strips_surrounding_whitespace(self):
|
|
directive = output_language_directive(" Korean ")
|
|
assert directive is not None
|
|
assert "Korean" in directive
|
|
assert " Korean" not in directive
|
|
|
|
|
|
class TestReplyLanguageDirective:
|
|
"""Precedence: explicit OUTPUT_LANGUAGE lock > English-only TTS > free.
|
|
|
|
The lock must override the Piper/Chatterbox English fallback, because a
|
|
deployment that sets OUTPUT_LANGUAGE (e.g. Korean) also runs a TTS voice
|
|
that can speak it. Without this, the English lock and the Korean lock
|
|
contradict each other and the model reverts to English.
|
|
"""
|
|
|
|
def test_lock_overrides_english_only_tts(self):
|
|
directive = reply_language_directive("Korean", "piper")
|
|
assert directive is not None
|
|
assert "Korean" in directive
|
|
assert directive != ENGLISH_ONLY_DIRECTIVE
|
|
|
|
def test_english_only_tts_forces_english_without_lock(self):
|
|
assert reply_language_directive(None, "piper") == ENGLISH_ONLY_DIRECTIVE
|
|
assert reply_language_directive("", "chatterbox") == ENGLISH_ONLY_DIRECTIVE
|
|
|
|
def test_no_lock_multilingual_tts_is_free(self):
|
|
# A non-English-only engine (e.g. melo) with no lock → reply in the
|
|
# user's own language, so no directive.
|
|
assert reply_language_directive(None, "melo") is None
|
|
|
|
def test_unknown_tts_defaults_to_english_only(self):
|
|
# Preserves the original getattr(cfg, 'tts_engine', 'piper') default:
|
|
# an unknown/missing engine is treated conservatively as English-only.
|
|
assert reply_language_directive(None, None) == ENGLISH_ONLY_DIRECTIVE
|
|
|
|
def test_lock_wins_even_with_multilingual_tts(self):
|
|
directive = reply_language_directive("Korean", "melo")
|
|
assert directive is not None and "Korean" in directive
|
|
|
|
def test_edge_is_multilingual(self):
|
|
# Edge TTS (the default Korean voice) is not English-only: no lock → the
|
|
# user's own language, and a lock is honoured (not forced to English).
|
|
assert reply_language_directive(None, "edge") is None
|
|
directive = reply_language_directive("Korean", "edge")
|
|
assert directive is not None and "Korean" in directive
|
|
assert directive != ENGLISH_ONLY_DIRECTIVE
|
|
|
|
|
|
class TestLoadAgentInstructions:
|
|
"""Operator can extend the reply LLM's system prompt by dropping *.md files
|
|
into an agents/ folder. The loader concatenates them in filename order and
|
|
fails open so a missing/empty/broken folder never breaks a reply."""
|
|
|
|
def test_missing_dir_returns_empty(self, tmp_path):
|
|
assert load_agent_instructions(str(tmp_path / "does-not-exist")) == ""
|
|
|
|
def test_empty_dir_returns_empty(self, tmp_path):
|
|
assert load_agent_instructions(str(tmp_path)) == ""
|
|
|
|
def test_reads_and_concatenates_single_file(self, tmp_path):
|
|
(tmp_path / "rules.md").write_text("Always be brief.", encoding="utf-8")
|
|
assert load_agent_instructions(str(tmp_path)) == "Always be brief."
|
|
|
|
def test_files_are_ordered_by_filename(self, tmp_path):
|
|
# Filename prefixes let the operator control ordering.
|
|
(tmp_path / "10-second.md").write_text("SECOND", encoding="utf-8")
|
|
(tmp_path / "00-first.md").write_text("FIRST", encoding="utf-8")
|
|
result = load_agent_instructions(str(tmp_path))
|
|
assert result.index("FIRST") < result.index("SECOND")
|
|
|
|
def test_only_md_files_are_read(self, tmp_path):
|
|
(tmp_path / "note.txt").write_text("IGNORE ME", encoding="utf-8")
|
|
(tmp_path / "use.md").write_text("USE ME", encoding="utf-8")
|
|
result = load_agent_instructions(str(tmp_path))
|
|
assert "USE ME" in result
|
|
assert "IGNORE ME" not in result
|
|
|
|
def test_blank_files_are_skipped(self, tmp_path):
|
|
(tmp_path / "blank.md").write_text(" \n ", encoding="utf-8")
|
|
(tmp_path / "real.md").write_text("Real instruction.", encoding="utf-8")
|
|
assert load_agent_instructions(str(tmp_path)) == "Real instruction."
|
|
|
|
def test_env_var_is_used_when_no_arg(self, tmp_path, monkeypatch):
|
|
(tmp_path / "a.md").write_text("FROM ENV", encoding="utf-8")
|
|
monkeypatch.setenv("AGENTS_DIR", str(tmp_path))
|
|
assert load_agent_instructions() == "FROM ENV"
|
|
|
|
def test_explicit_arg_overrides_env(self, tmp_path, monkeypatch):
|
|
(tmp_path / "env.md").write_text("ENV", encoding="utf-8")
|
|
other = tmp_path / "other"
|
|
other.mkdir()
|
|
(other / "arg.md").write_text("ARG", encoding="utf-8")
|
|
monkeypatch.setenv("AGENTS_DIR", str(tmp_path))
|
|
assert load_agent_instructions(str(other)) == "ARG"
|
|
|
|
def test_a_file_path_instead_of_dir_returns_empty(self, tmp_path):
|
|
f = tmp_path / "file.md"
|
|
f.write_text("x", encoding="utf-8")
|
|
# Pointed at a file, not a directory → fail-open.
|
|
assert load_agent_instructions(str(f)) == ""
|