diff --git a/docs/llm_contexts.md b/docs/llm_contexts.md index c26d291..05c2578 100644 --- a/docs/llm_contexts.md +++ b/docs/llm_contexts.md @@ -161,7 +161,7 @@ Every distinct LLM call in Jarvis, what feeds it, what consumes it, and how it i ## 12. Task-list Planner (pre-flight decomposition, gates the whole turn) - **File**: [src/jarvis/reply/planner.py](src/jarvis/reply/planner.py) — `plan_query()`. -- **Trigger**: once per reply, **after the tool router and before memory search**. **v2 brain** (`intent_judge=True`): runs only on turns where the router picked a tool (not for the sentinels `stop` or `toolSearchTool` alone; a router-picked `chatChannels` counts), so a turn without tools has one LLM call before its answer. Also skipped when the router's only pick is a one-step tool (`_ONE_STEP_TOOLS` in `engine.py`: `getWeather`, `logMeal`, `fetchMeals`, `screenView`) and the chat model calls tools natively; the chat model then gets the tool directly, and an answer it gives without the tool is dropped and followed by one nudge to call it (planner.spec.md, "Intent judge (v2)"). Skipped when `cfg.planner_enabled = False`, when the query is shorter than `MIN_QUERY_CHARS` (4), or when no model / base URL is available. +- **Trigger**: once per reply, **after the tool router and before memory search**. **v2 brain** (`intent_judge=True`): runs only on turns where the router picked a tool (not for the sentinels `stop` or `toolSearchTool` alone; a router-picked `chatChannels` counts), so a turn without tools has one LLM call before its answer. Also skipped when the router's only pick is a one-step tool (`_ONE_STEP_TOOLS` in `engine.py`: `getWeather`, `logMeal`, `fetchMeals`, `screenView`) and the chat model calls tools natively; the chat model then gets the tool directly, and an answer it gives without the tool is dropped and followed by one nudge to call it; a second answer without the tool runs the planner after all (one extra call, only then) and the skipped-tool guard applies (planner.spec.md, "Intent judge (v2)"). Skipped when `cfg.planner_enabled = False`, when the query is shorter than `MIN_QUERY_CHARS` (4), or when no model / base URL is available. - **Model / gating**: resolution chain `planner_model (override) → ollama_chat_model`. The planner tracks the chat model so upgrading the chat model (via setup wizard or config) automatically upgrades plan quality. - **Inputs**: user query, dialogue context, the live time/location line (`LIVE CONTEXT:`, so a location-dependent ask plans `searchMemory topic='where the user lives'` only when the location is unknown or disabled), **router-narrowed** tool catalogue (names + one-line descriptions + argument names, built by `planner_tool_catalog()`) — not the full 30+ list. When the carry-over guard from #7 fires, the previous turn's failed tool name is unioned into this catalogue before the planner sees it, so the planner can plan a re-call without `toolSearchTool` round-tripping. **No** memory context — the planner decides *whether* memory is needed. - **System prompt**: `_PROMPT_TEMPLATE` in `planner.py`. Teaches the `searchMemory topic='...'` directive for prior-conversation lookups, short imperative tool steps, angle-bracket entity placeholders, final synthesis step, same-language output (the words of search and play arguments too, repeated in the user message's closing line), no numbering. @@ -215,7 +215,7 @@ Every distinct LLM call in Jarvis, what feeds it, what consumes it, and how it i | 10 | Graph extraction | ~1/session | No (background) | LARGE | | 11 | Graph best-child | 0-N | No (background) | SMALL (via router chain) | | 11b | Graph node merge | 0-N (per node, batched) | No (background) | SMALL (via router chain) | -| 12 | Planner (plan_query) | 0-1 (v2: tool turns except one-step tools) | yes (planner_enabled) | LARGE/SMALL (tracks chat model) | +| 12 | Planner (plan_query) | 0-1 (v2: tool turns except one-step tools, which plan only when the model skips the tool twice) | yes (planner_enabled) | LARGE/SMALL (tracks chat model) | | 13 | Plan step resolver | 0-N (SMALL only) | auto by size + plan | SMALL (via router chain) | | 14 | Tool-specific | per-tool | n/a | LARGE | | 15 | Voice call judgements | 0 (per utterance / tie, outside the reply) | call mode / policy | LARGE (chat model) | diff --git a/src/jarvis/reply/engine.py b/src/jarvis/reply/engine.py index d0fdf84..cea1783 100644 --- a/src/jarvis/reply/engine.py +++ b/src/jarvis/reply/engine.py @@ -3027,6 +3027,36 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any], candidate_reply = content malformed_fallback = False + _one_step_skipped = bool( + _one_step_tool and _one_step_tool in allowed_tools and not use_text_tools + and not _forced_plan_steps and not malformed_fallback + and sum(1 for m in messages if m.get("tool_name")) == _plan_steps_baseline + ) + if _one_step_skipped and not _one_step_nudged: + messages.pop() + messages.append({"role": "user", "content": ( + f"[Before answering, call {_one_step_tool}: this request needs its result. " + "If a detail it needs is missing, ask the user for it instead.]")}) + _one_step_nudged = True + debug_log(f"one-step turn: answered without {_one_step_tool}; asking for the call", "planning") + continue + if _one_step_skipped and not action_plan: + # Asked once and still no tool: this turn takes the planned path after all, so the + # skipped-tool guard below runs the tool the plan names (or the plan says no tool is + # needed, a question back to the user say, and the answer goes out). + try: + action_plan = plan_query( + cfg=cfg, query=redacted, dialogue_context=_dialogue_ctx, + tools=_planner_tool_catalog, live_context=_live_time_location_string(cfg), + ) + except Exception as _plan_exc: # pragma: no cover — defensive + debug_log(f"planner step failed (non-fatal): {_plan_exc}", "planning") + action_plan = [] + if action_plan: + _turn_state.trace("plan", steps=list(action_plan)) + debug_log(f"one-step turn: still no {_one_step_tool} after the nudge; planned " + f"{len(action_plan)} step(s)", "planning") + _guard_steps = [] if (use_text_tools or _forced_plan_steps or malformed_fallback) else ( named_tool_steps( action_plan, @@ -3049,18 +3079,6 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any], ) print(" 🧭 Planned tool skipped by the model, running it directly", flush=True) continue - if ( - _one_step_tool and _one_step_tool in allowed_tools and not use_text_tools - and not _one_step_nudged and not malformed_fallback - and sum(1 for m in messages if m.get("tool_name")) == _plan_steps_baseline - ): - messages.pop() - messages.append({"role": "user", "content": ( - f"[Before answering, call {_one_step_tool}: this request needs its result. " - "If a detail it needs is missing, ask the user for it instead.]")}) - _one_step_nudged = True - debug_log(f"one-step turn: answered without {_one_step_tool}; asking for the call", "planning") - continue reply = candidate_reply last_candidate_reply = candidate_reply diff --git a/src/jarvis/reply/planner.spec.md b/src/jarvis/reply/planner.spec.md index ac6a12b..e96498f 100644 --- a/src/jarvis/reply/planner.spec.md +++ b/src/jarvis/reply/planner.spec.md @@ -234,8 +234,11 @@ direct execution runs the plan. Without a plan the skipped-tool guard above has nothing to run, so these turns get a nudge instead: when the chat model answers in words before any tool ran in this reply, the answer is dropped (no later turn sees it) and a note asks the model to call that tool first, or to ask the user for a -missing detail. The nudge is sent at most once per reply; a second answer without the tool (a -question back to the user, say) is delivered. +missing detail. The nudge is sent at most once per reply. When the model answers without the tool +again, that turn falls back to the planned path: the planner runs then (one extra planner call, only +on this failure), and the skipped-tool guard above treats the second answer as on any planned turn. +A plan naming the tool runs it and the made-up answer never goes out; a plan that needs no tool (a +question back to the user, say) lets the answer go out, the planner judging that as before. Measured on Strata (Qwen3.6, 2026-11-16 16:45-16:58, the engine log): the planner averaged 2.5 s (9 calls), one after the other before the answer on every tool turn, while the router runs during diff --git a/src/jarvis/reply/reply.spec.md b/src/jarvis/reply/reply.spec.md index 5b66a15..1a3f47b 100644 --- a/src/jarvis/reply/reply.spec.md +++ b/src/jarvis/reply/reply.spec.md @@ -106,7 +106,7 @@ Design principles enforced by the engine: - When detected, the engine falls back to the standard "I had trouble understanding that request" error reply (model-size-aware). The malformed content is never shown to the user. Skipped-tool guard (native tool calling): - - When a native-tool model answers in prose before any tool has run in this reply, and the plan names a callable tool as a concrete call, the engine discards that answer and direct-executes the plan's tool steps, then the model answers from the results. Only literal, schema-valid steps run (no LLM repair). When the planned call cannot run, the held-back answer is still discarded: the planned call is recorded as a failed tool result and the model answers from that (an honest "could not check or do it"). See `planner.spec.md` ("Skipped-tool guard"). v2 one-step turns (no plan) get one nudge to call the tool instead (`planner.spec.md`, "Intent judge (v2)"). + - When a native-tool model answers in prose before any tool has run in this reply, and the plan names a callable tool as a concrete call, the engine discards that answer and direct-executes the plan's tool steps, then the model answers from the results. Only literal, schema-valid steps run (no LLM repair). When the planned call cannot run, the held-back answer is still discarded: the planned call is recorded as a failed tool result and the model answers from that (an honest "could not check or do it"). See `planner.spec.md` ("Skipped-tool guard"). v2 one-step turns (no plan) get one nudge to call the tool instead, and when the model skips it again the turn is planned and guarded as above (`planner.spec.md`, "Intent judge (v2)"). Task-list planner (all model sizes, strongest impact on small models): - The planner runs at the **front** of the reply flow (see step 3 above), not after tool selection. By the time the agentic loop starts, the plan already exists, the memory block has either been run or skipped based on the plan's `searchMemory` directive, and the tool allow-list has been derived from the tool names the plan referenced. See `planner.spec.md` for the prompt contract and fail-open semantics. diff --git a/tests/test_engine_compact_prompt.py b/tests/test_engine_compact_prompt.py index bc5775e..7261481 100644 --- a/tests/test_engine_compact_prompt.py +++ b/tests/test_engine_compact_prompt.py @@ -53,7 +53,7 @@ def test_the_full_tool_guidance_comes_only_with_a_data_tool(): assert TOOL_CONSTRAINTS_LARGE in weather and COMPACT_ACTION_RULE not in weather -def _intent_turn(prepared, routed=("getWeather",)): +def _intent_turn(prepared, routed=("getWeather",), first=None): cfg = _mock_cfg(512) cfg.stream_browser = False cfg.tool_selection_strategy = "llm" @@ -62,7 +62,9 @@ def _intent_turn(prepared, routed=("getWeather",)): patch("src.jarvis.reply.engine._extra_config", lambda key, default="": default), \ patch("src.jarvis.reply.engine.extract_text_from_response", return_value="네."), \ patch("src.jarvis.reply.engine.chat_with_messages") as mock_chat: - mock_chat.return_value = {"message": {"content": "네."}} + answer = {"message": {"content": "네."}} + mock_chat.side_effect = [first, answer] if first else None + mock_chat.return_value = answer run_reply_engine(db=Mock(), cfg=cfg, tts=None, text="날씨 어때?", dialogue_memory=DialogueMemory(), history=HistoryContext(), legacy_memory=False, intent_judge=True, prepared_tools=prepared) @@ -77,8 +79,12 @@ def test_the_intent_judge_plans_only_turns_that_need_a_tool(): # A tool turn that may take several steps: the planner orders them. planner, router, tools = _intent_turn(None, routed=("webSearch",)) assert (planner, router) == (1, 1) and "webSearch" in tools - # A lone one-step tool needs no plan (tests/test_engine_one_step_turn.py). - planner, router, tools = _intent_turn(None) + # A lone one-step tool the model calls needs no plan (tests/test_engine_one_step_turn.py). + weather_call = {"message": {"content": "", "tool_calls": [ + {"id": "c1", "type": "function", "function": {"name": "getWeather", "arguments": {}}}]}} + with patch("src.jarvis.reply.engine.run_tool_with_retries", + return_value=Mock(success=True, reply_text="13C", error_message=None, images=None)): + planner, router, tools = _intent_turn(None, first=weather_call) assert (planner, router) == (0, 1) and "getWeather" in tools diff --git a/tests/test_engine_one_step_turn.py b/tests/test_engine_one_step_turn.py index fb4c978..29006d4 100644 --- a/tests/test_engine_one_step_turn.py +++ b/tests/test_engine_one_step_turn.py @@ -64,7 +64,7 @@ def _run(mock_config, db, dialogue_memory, *, routed, chat_replies, plan=("Reply @pytest.mark.parametrize("routed", [["getWeather", "stop"], ["logMeal", "stop"], ["screenView"]]) def test_a_lone_one_step_tool_needs_no_plan(mock_config, db, dialogue_memory, routed): _, _, _, planned = _run(mock_config, db, dialogue_memory, routed=routed, - chat_replies=[_content(GROUNDED)] * 2) + chat_replies=[_native_call(routed[0], {}), _content(GROUNDED)]) assert planned == 0 @@ -94,12 +94,27 @@ def test_a_tool_less_answer_on_a_one_step_turn_is_dropped_and_the_tool_called(mo @pytest.mark.unit -def test_the_tool_is_asked_for_only_once(mock_config, db, dialogue_memory): - # A second tool-less answer (for example a question back to the user) is delivered. +def test_a_second_made_up_answer_falls_back_to_the_plan_and_its_tool(mock_config, db, dialogue_memory): + # Asked once, the model still answers without the tool: the turn takes the planned path, the + # skipped-tool guard runs the planned call, and no made-up answer reaches the user. + reply, invoked, seen, planned = _run( + mock_config, db, dialogue_memory, routed=["getWeather", "stop"], + plan=["getWeather location='Seoul'", "Reply to the user with the weather."], + chat_replies=[_content(FABRICATED), _content(FABRICATED), _content(GROUNDED)]) + assert planned == 1 + assert invoked == ["getWeather"] + assert reply == GROUNDED + assert not any(FABRICATED in (m.get("content") or "") for m in seen[-1]) + + +@pytest.mark.unit +def test_a_question_back_goes_out_when_the_plan_needs_no_tool(mock_config, db, dialogue_memory): + # The late plan judges that no tool is needed (a question back to the user): it is delivered. question = "어느 도시 날씨를 알려 드릴까요?" - reply, invoked, seen, _ = _run( + reply, invoked, seen, planned = _run( mock_config, db, dialogue_memory, routed=["getWeather", "stop"], chat_replies=[_content(question), _content(question)]) + assert planned == 1 assert invoked == [] assert reply == question assert len(seen) == 2