Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
83999a5b0b | ||
|
|
d5fd218c86 |
@@ -22,11 +22,15 @@ class ModelSize(Enum):
|
|||||||
LARGE = "large" # 8b+ - can infer tool usage from context
|
LARGE = "large" # 8b+ - can infer tool usage from context
|
||||||
|
|
||||||
|
|
||||||
# Model size patterns - models matching these are considered SMALL
|
# Model size patterns - models matching these are considered SMALL.
|
||||||
|
# Covers every sub-8B size (1b-7b): these models need the explicit, repeated
|
||||||
|
# tool/greeting/instruction constraints and falter on the terse LARGE prompt.
|
||||||
|
# Without 2b/4b/5b/6b here a genuinely small model (e.g. qwen*:4b) was
|
||||||
|
# misclassified as LARGE and given the less-guided prompt set.
|
||||||
_SMALL_MODEL_PATTERNS = (
|
_SMALL_MODEL_PATTERNS = (
|
||||||
":1b", ":3b", ":7b",
|
":1b", ":2b", ":3b", ":4b", ":5b", ":6b", ":7b",
|
||||||
"-1b", "-3b", "-7b",
|
"-1b", "-2b", "-3b", "-4b", "-5b", "-6b", "-7b",
|
||||||
"_1b", "_3b", "_7b",
|
"_1b", "_2b", "_3b", "_4b", "_5b", "_6b", "_7b",
|
||||||
"gemma4", # Gemma 4 - always small regardless of tag
|
"gemma4", # Gemma 4 - always small regardless of tag
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ This module provides model-size-aware prompt generation for the reply engine.
|
|||||||
|
|
||||||
### Problem Statement
|
### Problem Statement
|
||||||
|
|
||||||
Small models (1b, 3b, 7b parameters) lack the reasoning capacity to infer when NOT to use tools. When given prompts like "Proactively use available tools," they may incorrectly call tools for simple greetings like "hello" or "ni hao" because they cannot distinguish between:
|
Small models (every sub-8B size, 1b-7b parameters) lack the reasoning capacity to infer when NOT to use tools. When given prompts like "Proactively use available tools," they may incorrectly call tools for simple greetings like "hello" or "ni hao" because they cannot distinguish between:
|
||||||
- Requests that require tools (weather, search, data retrieval)
|
- Requests that require tools (weather, search, data retrieval)
|
||||||
- Simple conversation (greetings, small talk, general knowledge)
|
- Simple conversation (greetings, small talk, general knowledge)
|
||||||
|
|
||||||
@@ -14,7 +14,7 @@ The module detects model size from the model name and selects appropriate prompt
|
|||||||
|
|
||||||
| Model Size | Detection Pattern | Tool Prompts |
|
| Model Size | Detection Pattern | Tool Prompts |
|
||||||
|------------|-------------------|--------------|
|
|------------|-------------------|--------------|
|
||||||
| SMALL | `:1b`, `:3b`, `:7b`, `gemma4` | Conservative — explicit "DO NOT use tools for greetings" + worked negative examples + repetition |
|
| SMALL | `:1b`-`:7b` (every size 1-7B, all separators), `gemma4` | Conservative — explicit "DO NOT use tools for greetings" + worked negative examples + repetition |
|
||||||
| LARGE | All others (8b+) | Proactive — "use tools confidently" + short anti-confabulation + auto-derive clause |
|
| LARGE | All others (8b+) | Proactive — "use tools confidently" + short anti-confabulation + auto-derive clause |
|
||||||
|
|
||||||
### Architecture
|
### Architecture
|
||||||
|
|||||||
@@ -120,10 +120,13 @@ _SYSTEM_PROMPT_TEMPLATE: str = (
|
|||||||
"'tell me a joke', 'chat with me'), never reply with a bare greeting like 'Hey there!', "
|
"'tell me a joke', 'chat with me'), never reply with a bare greeting like 'Hey there!', "
|
||||||
"'Hi!', 'How can I help you?', or a generic observation about an unrelated topic. "
|
"'Hi!', 'How can I help you?', or a generic observation about an unrelated topic. "
|
||||||
"When the 'Information the user has shared…' section is present, you MUST pick one concrete "
|
"When the 'Information the user has shared…' section is present, you MUST pick one concrete "
|
||||||
"fact from it and build the reply around that fact (e.g. 'You mentioned you box at Trenches "
|
"fact from it and build the reply around that fact, opening with a short natural reference to "
|
||||||
"Gym — how's training going this week?'). Do not talk about things that are not in that "
|
"it. CRITICAL: use ONLY names, places, activities, and details that literally appear in that "
|
||||||
"section. Only when that section is absent may you invent a fresh observation, question, or "
|
"section — never borrow any name, place, or activity from these instructions or from any "
|
||||||
"joke. Produce a varied response each time — do not repeat a previous reply verbatim. "
|
"example wording, and never invent specifics that are not in that section. Do not talk about "
|
||||||
|
"things that are not in that section. Only when that section is absent may you invent a fresh "
|
||||||
|
"observation, question, or joke. Produce a varied response each time — do not repeat a "
|
||||||
|
"previous reply verbatim. "
|
||||||
"Banned phrasings: 'I can only tell you what you have shared with me in this conversation', "
|
"Banned phrasings: 'I can only tell you what you have shared with me in this conversation', "
|
||||||
"'I don't have access to any personal information outside of what you tell me', 'I don't have "
|
"'I don't have access to any personal information outside of what you tell me', 'I don't have "
|
||||||
"personal details outside of our conversation history', 'I do not store personal details "
|
"personal details outside of our conversation history', 'I do not store personal details "
|
||||||
|
|||||||
@@ -21,9 +21,15 @@ class TestModelSizeDetection:
|
|||||||
("gemma:7b", True),
|
("gemma:7b", True),
|
||||||
("phi3:3b", True),
|
("phi3:3b", True),
|
||||||
("qwen2:7b", True),
|
("qwen2:7b", True),
|
||||||
|
# Sub-8B sizes that were previously misclassified as LARGE.
|
||||||
|
("qwen3.5:4b", True), # the deployed model that produced weak, off-tone replies
|
||||||
|
("gemma2:2b", True),
|
||||||
|
("model:5b", True),
|
||||||
|
("model:6b", True),
|
||||||
# Various separators
|
# Various separators
|
||||||
("model-3b-instruct", True),
|
("model-3b-instruct", True),
|
||||||
("model_1b_chat", True),
|
("model_1b_chat", True),
|
||||||
|
("model-4b-instruct", True),
|
||||||
# Large models (should return LARGE)
|
# Large models (should return LARGE)
|
||||||
("gpt-oss:20b", False),
|
("gpt-oss:20b", False),
|
||||||
("llama3.1:8b", False),
|
("llama3.1:8b", False),
|
||||||
|
|||||||
@@ -60,6 +60,16 @@ class TestBuildSystemPrompt:
|
|||||||
assert "SINGLE sentence" in prompt
|
assert "SINGLE sentence" in prompt
|
||||||
assert "never add it as " in prompt
|
assert "never add it as " in prompt
|
||||||
|
|
||||||
|
def test_persona_has_no_copyable_proper_noun_examples(self):
|
||||||
|
# A weak model parroted the literal "Trenches Gym" example from the
|
||||||
|
# persona as if it were a real user fact (boxing mangled to tennis).
|
||||||
|
# The persona must not embed copyable personal proper nouns, and must
|
||||||
|
# tell the model to use ONLY facts that literally appear in the memory
|
||||||
|
# section — never borrow names/places from the instructions themselves.
|
||||||
|
prompt = build_system_prompt("Jarvis")
|
||||||
|
assert "Trenches" not in prompt
|
||||||
|
assert "never borrow any name, place, or activity from these instructions" in prompt
|
||||||
|
|
||||||
|
|
||||||
class TestOutputLanguageDirective:
|
class TestOutputLanguageDirective:
|
||||||
"""A deployment may lock replies to a single language via OUTPUT_LANGUAGE.
|
"""A deployment may lock replies to a single language via OUTPUT_LANGUAGE.
|
||||||
|
|||||||
Reference in New Issue
Block a user