perf: run auxiliary LLM calls on a small model, big model only for the answer

Intent judging, tool routing and arg extraction are classification/JSON calls,
not the spoken answer, yet the stack aliased OLLAMA_INTENT_MODEL back to the big
chat model — so each command paid the big model's cost 2-3 times for routing
before the reply even ran. With the GPU on, that round-trip stacking is the main
remaining per-turn latency. Default OLLAMA_INTENT_MODEL to qwen2.5:3b (the
project's reference small model, clean Korean on classification) and have
ollama-init pull it. The reply engine already routes these calls through
intent_judge_model, so answer quality is untouched; set OLLAMA_INTENT_MODEL =
OLLAMA_CHAT_MODEL to fold back onto one resident model.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

.env.example

@@ -59,11 +59,12 @@ OLLAMA_BASE_URL=http://127.0.0.1:11434
 # free-form chit-chat. Swap back to qwen3:8b for the strongest tool-calling.
 OLLAMA_CHAT_MODEL=qwen2.5:3b
 # Model for the auxiliary small-model calls: intent judge, tool router, weather
-# place extraction, query decomposition. BLANK (default) reuses OLLAMA_CHAT_MODEL
-# so the stack runs on one already-warm model. The code's built-in default
-# (gemma4:e2b) is NOT pulled by this stack, so leaving this unset previously made
-# every router/extractor call silently fail. Only set this if you also pull the
-# model into Ollama.
+# place extraction, query decomposition. These are classification/JSON calls,
+# NOT the spoken answer, so a small fast model here cuts 2-3 big-model round
+# trips per command without touching answer quality. BLANK uses the stack
+# default qwen2.5:3b, which ollama-init pulls automatically. Set it equal to
+# OLLAMA_CHAT_MODEL to run everything on one resident model instead (saves VRAM
+# at the cost of slower routing when the chat model is large).
 OLLAMA_INTENT_MODEL=
 OLLAMA_EMBED_MODEL=nomic-embed-text
 WHISPER_MODEL=small

bot/scripts/stream-test/browse-search.mjs

@@ -2,10 +2,11 @@
 // 9222) so the action is visible on the Go-Live broadcast, and prints a JSON
 // result on stdout for the Python `browseAndSearch` tool to wrap.
 //
-//   node browse-search.mjs "<query>" [search|youtube]
+//   node browse-search.mjs "<query>" [search|youtube] [index]
 //
 // - search  : Google-search the query, return the top organic results.
-// - youtube : search YouTube and play the first result.
+// - youtube : search YouTube and play a result. `index` is the 1-based position
+//             from the top of the result list (default 1 = first result).
 //
 // Backend selection for `search`:
 //   1. The broadcast Chrome over CDP (visible on the Go-Live stream).
@@ -29,6 +30,9 @@ const UA =
   '(KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36';
 const query = process.argv[2] || '';
 const mode = (process.argv[3] || 'search').toLowerCase();
+// 1-based position of the YouTube result to play, counted from the top of the
+// list. Defaults to 1 (first result). Anything <1 or non-numeric falls back to 1.
+const playIndex = Math.max(1, parseInt(process.argv[4], 10) || 1);
 const out = (o) => { process.stdout.write(JSON.stringify(o)); };
 
 if (!query) { out({ ok: false, error: 'no query' }); process.exit(1); }
@@ -105,15 +109,21 @@ try {
   await page.bringToFront().catch(() => {});
 
   if (mode === 'youtube') {
-    // Type into YouTube's search box like a person, then play the first result.
+    // Type into YouTube's search box like a person, then play the requested
+    // result (the Nth from the top of the list; default the first).
     await typeSearch('https://www.youtube.com/?hl=ko', 'input#search, input[name="search_query"]', query);
     await page.waitForSelector('ytd-video-renderer a#video-title, a#video-title', { timeout: 20000 });
-    const first = page.locator('ytd-video-renderer a#video-title, a#video-title').first();
-    const title = (await first.getAttribute('title').catch(() => '')) || (await first.innerText().catch(() => ''));
-    await first.click();
+    const results = page.locator('ytd-video-renderer a#video-title, a#video-title');
+    // Clamp to what's actually on the page so "play the 5th" still plays the
+    // last available result rather than failing when fewer were returned.
+    const available = await results.count();
+    const targetIdx = Math.min(playIndex, Math.max(available, 1)) - 1;
+    const target = results.nth(targetIdx);
+    const title = (await target.getAttribute('title').catch(() => '')) || (await target.innerText().catch(() => ''));
+    await target.click();
     await page.waitForSelector('#movie_player', { timeout: 20000 });
     await page.evaluate(() => { const v = document.querySelector('video'); if (v && v.paused) v.play(); });
-    out({ ok: true, mode, title: (title || '').trim(), url: page.url() });
+    out({ ok: true, mode, index: targetIdx + 1, title: (title || '').trim(), url: page.url() });
   } else {
     // Type into Google's search box like a person, then read the results.
     await typeSearch('https://www.google.com/?hl=ko', 'textarea[name="q"], input[name="q"]', query);

docker-compose.yml

@@ -40,6 +40,9 @@ services:
     environment:
       OLLAMA_HOST: http://ollama:11434
       CHAT_MODEL: ${OLLAMA_CHAT_MODEL:-qwen2.5:3b}
+      # Small auxiliary model for intent/router/extraction calls (see javis
+      # service). Pulled here so the split is ready out of the box.
+      INTENT_MODEL: ${OLLAMA_INTENT_MODEL:-qwen2.5:3b}
       EMBED_MODEL: ${OLLAMA_EMBED_MODEL:-nomic-embed-text}
     entrypoint: ["/bin/sh", "-c"]
     command:
@@ -48,6 +51,10 @@ services:
         until ollama list >/dev/null 2>&1; do sleep 2; done;
         echo "[ollama-init] pulling $$CHAT_MODEL";
         ollama pull "$$CHAT_MODEL";
+        if [ -n "$$INTENT_MODEL" ] && [ "$$INTENT_MODEL" != "$$CHAT_MODEL" ]; then
+          echo "[ollama-init] pulling $$INTENT_MODEL (auxiliary intent/router model)";
+          ollama pull "$$INTENT_MODEL";
+        fi;
         echo "[ollama-init] pulling $$EMBED_MODEL";
         ollama pull "$$EMBED_MODEL";
         echo "[ollama-init] models ready.";
@@ -62,6 +69,14 @@ services:
       # Point the brain at the ollama service and the bot at the in-container bridge.
       OLLAMA_BASE_URL: http://ollama:11434
       OLLAMA_CHAT_MODEL: ${OLLAMA_CHAT_MODEL:-qwen2.5:3b}
+      # Auxiliary small-model calls (intent judge, tool router, arg extraction,
+      # query decomposition) run on this fast model so the big chat model only
+      # runs for the actual spoken answer. With the GPU on, this is the main
+      # per-turn latency win: a command no longer pays the big model's cost 2-3
+      # times for routing/extraction. Defaults to qwen2.5:3b (the project's
+      # reference small model, clean Korean on classification); set it equal to
+      # OLLAMA_CHAT_MODEL to fold everything back onto one resident model.
+      OLLAMA_INTENT_MODEL: ${OLLAMA_INTENT_MODEL:-qwen2.5:3b}
       OLLAMA_EMBED_MODEL: ${OLLAMA_EMBED_MODEL:-nomic-embed-text}
       WHISPER_MODEL: ${WHISPER_MODEL:-medium}
       WHISPER_DEVICE: ${WHISPER_DEVICE:-cuda}

docker/entrypoint.sh

@@ -10,12 +10,15 @@ set -euo pipefail
 : "${OLLAMA_BASE_URL:=http://ollama:11434}"
 : "${OLLAMA_CHAT_MODEL:=qwen3:8b}"
 # Auxiliary small-model calls (intent judge, tool router, weather place
-# extraction, query decomposition). The code default is gemma4:e2b, which this
-# stack does not pull, so those calls would silently fail and fall open —
-# crippling tool routing and arg extraction. Reuse the (already warm) chat model
-# by default so everything runs on one resident model; override if you pull a
-# dedicated small model.
-: "${OLLAMA_INTENT_MODEL:=${OLLAMA_CHAT_MODEL}}"
+# extraction, query decomposition). Default to a small fast model so the big
+# chat model only runs for the actual spoken answer — the main per-turn latency
+# win once the GPU is in use, since the 2-3 routing/extraction calls per command
+# no longer pay the big model's cost. ollama-init pulls this model. Set it equal
+# to OLLAMA_CHAT_MODEL to fold everything back onto one resident model.
+: "${OLLAMA_INTENT_MODEL:=qwen2.5:3b}"
+# Cap chat-model output tokens per turn (worst-case latency guard). Spoken
+# answers are 1-2 sentences; 512 is safe headroom above tool-call JSON. 0 = off.
+: "${OLLAMA_NUM_PREDICT:=512}"
 : "${OLLAMA_EMBED_MODEL:=nomic-embed-text}"
 : "${WHISPER_MODEL:=small}"
 : "${WHISPER_DEVICE:=cuda}"
@@ -32,7 +35,7 @@ set -euo pipefail
 : "${XDG_RUNTIME_DIR:=/run/user/0}"
 : "${PULSE_SERVER:=unix:${XDG_RUNTIME_DIR}/pulse/native}"
 
-export VNC_RESOLUTION OLLAMA_BASE_URL OLLAMA_CHAT_MODEL OLLAMA_INTENT_MODEL OLLAMA_EMBED_MODEL \
+export VNC_RESOLUTION OLLAMA_BASE_URL OLLAMA_CHAT_MODEL OLLAMA_NUM_PREDICT OLLAMA_INTENT_MODEL OLLAMA_EMBED_MODEL \
        WHISPER_MODEL WHISPER_DEVICE WHISPER_COMPUTE_TYPE JARVIS_DB_PATH \
        PIPER_VOICE PIPER_VOICE_DIR TTS_PIPER_MODEL_PATH BRIDGE_HOST BRIDGE_PORT \
        XDG_RUNTIME_DIR PULSE_SERVER

docker/jarvis-config.template.json

@@ -4,6 +4,7 @@
   "ollama_base_url": "${OLLAMA_BASE_URL}",
   "ollama_embed_model": "${OLLAMA_EMBED_MODEL}",
   "ollama_chat_model": "${OLLAMA_CHAT_MODEL}",
+  "ollama_num_predict": "${OLLAMA_NUM_PREDICT}",
   "intent_judge_model": "${OLLAMA_INTENT_MODEL}",
   "tts_enabled": true,
   "tts_engine": "${TTS_ENGINE}",

docs/llm_contexts.md

@@ -20,7 +20,7 @@ Every distinct LLM call in Jarvis, what feeds it, what consumes it, and how it i
   - Tool schema: native via `generate_tools_json_schema()` ([src/jarvis/tools/registry.py](src/jarvis/tools/registry.py)) or text fallback via `_text_tool_call_guidance()` ([engine.py:68](src/jarvis/reply/engine.py:68))
   - Tool results from prior turns (raw or digested — see #5)
 - **Output**: OpenAI-style `{content, tool_calls, thinking}`. Consumed by the tool orchestrator and TTS pipeline. Natural-language content is delivered immediately; no post-turn evaluator runs.
-- **Limits**: `num_ctx: 8192` (explicit). Timeout `llm_chat_timeout_sec` (45s). Auto-fallback from native to text tool-calls on HTTP 400 (`ToolsNotSupportedError`), sticky for the session. Risk: `fetch_web_page` truncates at 50,000 chars (~37k tokens) — mitigated for SMALL models by tool-result digest (#5) which compresses the payload before it enters the messages history. LARGE models receive the raw payload and may silently see a truncated context.
+- **Limits**: `num_ctx: 8192` (explicit). Output `num_predict: cfg.ollama_num_predict` (default 512, `0`/negative disables) caps generated tokens per turn — a worst-case latency guard for short spoken answers; the headroom stays above tool-call JSON so it does not truncate tool calls (both native and text tool-call paths). Timeout `llm_chat_timeout_sec` (45s). Auto-fallback from native to text tool-calls on HTTP 400 (`ToolsNotSupportedError`), sticky for the session. Risk: `fetch_web_page` truncates at 50,000 chars (~37k tokens) — mitigated for SMALL models by tool-result digest (#5) which compresses the payload before it enters the messages history. LARGE models receive the raw payload and may silently see a truncated context.
 
 ## 2. Intent Judge
 

src/jarvis/config.py

@@ -85,6 +85,12 @@ class Settings:
     llm_digest_timeout_sec: float
     llm_embedding_timeout_sec: float
     llm_profile_select_timeout_sec: float
+    # Upper bound on tokens the chat model may generate per reply turn. Spoken
+    # (TTS) answers are 1-2 sentences, so a cap bounds the worst-case latency of
+    # a model that occasionally rambles or loops without changing normal answers.
+    # The headroom (default 512) sits well above this app's short tool-call JSON,
+    # so capping never truncates a tool call. 0 (or negative) disables the cap.
+    ollama_num_predict: int
 
     # Profiles & Behavior
     active_profiles: list[str]
@@ -394,6 +400,9 @@ def get_default_config() -> Dict[str, Any]:
         "llm_digest_timeout_sec": 8.0,
         "llm_embedding_timeout_sec": 60.0,
         "llm_profile_select_timeout_sec": 30.0,
+        # Cap on chat-model output tokens per turn (worst-case latency guard).
+        # 512 is safe headroom above short TTS answers and tool-call JSON; 0 off.
+        "ollama_num_predict": 512,
 
         # Profiles & Behavior
         "active_profiles": ["developer", "business", "life"],
@@ -763,6 +772,10 @@ def load_settings() -> Settings:
     llm_digest_timeout_sec = float(merged.get("llm_digest_timeout_sec", 8.0))
     llm_embedding_timeout_sec = float(merged.get("llm_embedding_timeout_sec", 60.0))
     llm_profile_select_timeout_sec = float(merged.get("llm_profile_select_timeout_sec", 30.0))
+    try:
+        ollama_num_predict = int(merged.get("ollama_num_predict", 512))
+    except (TypeError, ValueError):
+        ollama_num_predict = 512
 
     return Settings(
         # Database & Storage
@@ -778,6 +791,7 @@ def load_settings() -> Settings:
         llm_digest_timeout_sec=llm_digest_timeout_sec,
         llm_embedding_timeout_sec=llm_embedding_timeout_sec,
         llm_profile_select_timeout_sec=llm_profile_select_timeout_sec,
+        ollama_num_predict=ollama_num_predict,
 
         # Profiles & Behavior
         active_profiles=active_profiles,

src/jarvis/reply/engine.py

@@ -2233,6 +2233,16 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
                 has_tool_calls = " (has tool_calls)" if msg.get("tool_calls") else ""
                 debug_log(f"    [{i}] {role}: {content}{has_tool_calls}", "planning")
 
+        # Bound worst-case generation latency: spoken answers are 1-2 sentences,
+        # so cap the chat model's output tokens. The default headroom sits well
+        # above this app's tool-call JSON, so capping never truncates a tool
+        # call. 0/negative disables it. See config.ollama_num_predict.
+        try:
+            _num_predict = int(getattr(cfg, 'ollama_num_predict', 0) or 0)
+        except (TypeError, ValueError):
+            _num_predict = 0
+        _chat_extra_options = {"num_predict": _num_predict} if _num_predict > 0 else None
+
         # Send messages to Ollama — try native tool calling first, fall back to text-based
         # if the model returns HTTP 400 (native tools API not supported).
         _dump_tools_schema = None if use_text_tools else tools_json_schema
@@ -2242,7 +2252,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
                 chat_model=cfg.ollama_chat_model,
                 messages=messages,
                 timeout_sec=float(getattr(cfg, 'llm_chat_timeout_sec', 45.0)),
-                extra_options=None,
+                extra_options=_chat_extra_options,
                 tools=_dump_tools_schema,
                 thinking=getattr(cfg, 'llm_thinking_enabled', False),
             )
@@ -2273,7 +2283,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
                 chat_model=cfg.ollama_chat_model,
                 messages=messages,
                 timeout_sec=float(getattr(cfg, 'llm_chat_timeout_sec', 45.0)),
-                extra_options=None,
+                extra_options=_chat_extra_options,
                 tools=None,
                 thinking=getattr(cfg, 'llm_thinking_enabled', False),
             )

src/jarvis/reply/reply.spec.md

@@ -287,6 +287,8 @@ Turn 4: LLM → {content: "Here's a comprehensive comparison of the iPhone 15 mo
   - `llm_tools_timeout_sec` (enrichment extraction)
   - `llm_embed_timeout_sec` (vector search)
   - `llm_chat_timeout_sec` (messages loop turn)
+- Output bound:
+  - `ollama_num_predict` (default `512`, `0`/negative disables) caps the chat model's generated tokens per turn via the Ollama `num_predict` option on the messages-loop call. Spoken (TTS) answers are 1-2 sentences, so this never clips a normal answer; it bounds the worst-case latency of a model that occasionally rambles or loops. The default headroom sits well above this app's short tool-call JSON, so it does not truncate tool calls. Applied uniformly to the reply loop's chat call (both native-tools and text-tools paths); the small classification passes (intent judge, digests) keep their own caps. Note: this is a worst-case guard, not the primary latency lever, which is model size and GPU residency.
 - Memory enrichment:
   - `memory_enrichment_max_results` limits recalled snippets.
   - `memory_digest_enabled` (default `null` = auto-on for SMALL models ≤7B, off for LARGE) distils the combined diary + graph dump into a short relevance-filtered note via a cheap LLM pass before injecting into the system prompt. See **Memory Digest for Small Models** below.

src/jarvis/tools/builtin/browse_and_play.py

@@ -30,8 +30,10 @@ class BrowseAndPlayTool(Tool):
     def description(self) -> str:
         return (
             "Play a song / music video / clip on the shared screen by searching YouTube "
-            "and playing the first result. Use when the user asks you to play or watch "
-            "something. Only available in screen-share mode."
+            "and playing a result. Use when the user asks you to play or watch "
+            "something. Plays the first result by default; pass 'index' to play the "
+            "Nth result from the top of the search list (e.g. 'play the 3rd video' -> "
+            "index=3). Only available in screen-share mode."
         )
 
     @property
@@ -42,7 +44,16 @@ class BrowseAndPlayTool(Tool):
                 "query": {
                     "type": "string",
                     "description": "What to play, e.g. 'IU Good Day' or 'lofi hip hop'.",
-                }
+                },
+                "index": {
+                    "type": "integer",
+                    "description": (
+                        "1-based position of the video to play in the search results, "
+                        "counted from the top of the list. Defaults to 1 (first result). "
+                        "Use for 'play the Nth video' / 'play the second one'."
+                    ),
+                    "minimum": 1,
+                },
             },
             "required": ["query"],
         }
@@ -55,18 +66,25 @@ class BrowseAndPlayTool(Tool):
                 reply_text="화면 공유 모드(STREAM_BROWSER=true)에서만 영상을 재생할 수 있습니다.",
             )
         query = ""
+        index = 1
         if args and isinstance(args, dict):
             query = str(args.get("query", "")).strip()
+            try:
+                index = int(args.get("index", 1) or 1)
+            except (TypeError, ValueError):
+                index = 1
+            if index < 1:
+                index = 1
         if not query:
             return ToolExecutionResult(success=False, reply_text="재생할 내용을 알려주세요.")
         if not _NODE_SCRIPT.exists():
             return ToolExecutionResult(success=False, reply_text="브라우저 재생 도구를 찾을 수 없습니다.")
 
-        context.user_print(f"▶️ 화면에서 '{query}' 재생 중…")
-        debug_log(f"    ▶️ browseAndPlay '{query}'", "tools")
+        context.user_print(f"▶️ 화면에서 '{query}' 재생 중… (#{index})")
+        debug_log(f"    ▶️ browseAndPlay '{query}' index={index}", "tools")
         try:
             proc = subprocess.run(
-                ["node", str(_NODE_SCRIPT), query, "youtube"],
+                ["node", str(_NODE_SCRIPT), query, "youtube", str(index)],
                 capture_output=True,
                 text=True,
                 timeout=40,

src/jarvis/tools/builtin/browse_and_play.spec.md

@@ -6,16 +6,24 @@ video, or clip.
 
 ### Behaviour
 
-- Public schema is a single required `query` string (what to play).
+- Public schema is a required `query` string (what to play) plus an optional
+  `index` integer (1-based position in the search results, counted from the top
+  of the list). `index` defaults to `1` (first result), so existing callers and
+  "play X" requests are unchanged; "play the 3rd video" / "play the second one"
+  map to `index=3` / `index=2`.
 - **Mode-gated**: only acts when `STREAM_BROWSER` is true (`cfg.stream_browser`).
   In voice-only mode (false) there is no screen to show, so it returns a short
   message and does nothing.
 - Drives the on-screen Chrome by subprocessing the Node CDP helper
-  `bot/scripts/stream-test/browse-search.mjs <query> youtube`, which searches
-  YouTube and plays the first result on display `:1`. The broadcast captures
-  that display, so the playback is what viewers see.
-- Returns `success` with the played video's title, or a failure message if the
-  helper/Chrome is unavailable. It does NOT make an LLM call.
+  `bot/scripts/stream-test/browse-search.mjs <query> youtube <index>`, which
+  searches YouTube and plays the chosen result on display `:1`. The broadcast
+  captures that display, so the playback is what viewers see.
+  - The helper clicks the `index`-th `a#video-title` in the results list. The
+    index is clamped to the number of results actually returned, so asking for a
+    position beyond the list plays the last available result rather than failing.
+- Returns `success` with the played video's title (and the resolved `index`), or
+  a failure message if the helper/Chrome is unavailable. It does NOT make an LLM
+  call.
 
 ### Principles
 

tests/test_browse_and_play_index.py

@@ -0,0 +1,79 @@
+"""Tests for browseAndPlay's ``index`` argument (play the Nth search result).
+
+Behaviour verified:
+- default plays the first result (index 1) and stays backward-compatible,
+- an explicit index is forwarded to the Node helper as the 4th argv,
+- bad / sub-1 index values clamp to 1,
+- the index is advertised in the tool schema.
+"""
+
+import json
+from unittest.mock import Mock, patch
+
+import pytest
+
+from src.jarvis.tools.builtin.browse_and_play import BrowseAndPlayTool, _NODE_SCRIPT
+
+
+def _ctx():
+    cfg = Mock()
+    cfg.stream_browser = True
+    return Mock(cfg=cfg, user_print=Mock())
+
+
+def _run(args):
+    tool = BrowseAndPlayTool()
+    with patch("src.jarvis.tools.builtin.browse_and_play.subprocess.run") as mock_run:
+        mock_run.return_value = Mock(
+            stdout=json.dumps({"ok": True, "title": "Some Video"}),
+            stderr="",
+        )
+        result = tool.run(args, _ctx())
+        return mock_run, result
+
+
+def _argv(mock_run):
+    return list(mock_run.call_args[0][0])
+
+
+@pytest.mark.unit
+def test_schema_exposes_index():
+    schema = BrowseAndPlayTool().inputSchema
+    assert "index" in schema["properties"]
+    assert schema["properties"]["index"]["type"] == "integer"
+    assert "query" in schema["required"]
+    assert "index" not in schema["required"]  # optional
+
+
+@pytest.mark.unit
+def test_default_index_is_first():
+    mock_run, result = _run({"query": "IU Good Day"})
+    argv = _argv(mock_run)
+    assert argv[:4] == ["node", str(_NODE_SCRIPT), "IU Good Day", "youtube"]
+    assert argv[4] == "1"
+    assert result.success is True
+
+
+@pytest.mark.unit
+def test_explicit_index_forwarded():
+    mock_run, _ = _run({"query": "lofi", "index": 3})
+    assert _argv(mock_run)[4] == "3"
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize("bad", [0, -2, "nope", None])
+def test_bad_index_clamps_to_one(bad):
+    mock_run, _ = _run({"query": "lofi", "index": bad})
+    assert _argv(mock_run)[4] == "1"
+
+
+@pytest.mark.unit
+def test_voice_only_mode_does_not_play():
+    tool = BrowseAndPlayTool()
+    cfg = Mock()
+    cfg.stream_browser = False
+    ctx = Mock(cfg=cfg, user_print=Mock())
+    with patch("src.jarvis.tools.builtin.browse_and_play.subprocess.run") as mock_run:
+        result = tool.run({"query": "x", "index": 2}, ctx)
+    assert result.success is False
+    mock_run.assert_not_called()

tests/test_intent_model_split.py

@@ -0,0 +1,62 @@
+"""The docker deployment must run auxiliary calls on a small model.
+
+Latency win: intent judging, tool routing and arg extraction are
+classification/JSON calls, not the spoken answer. Running them on a small fast
+model means the big chat model only runs once per command (for the answer),
+instead of 2-3 times per command for routing/extraction too.
+
+The wiring is: docker/jarvis-config.template.json renders `intent_judge_model`
+from `${OLLAMA_INTENT_MODEL}`, and the reply engine's resolver falls through
+`tool_router_model -> intent_judge_model -> ollama_chat_model`. The template
+sets no `tool_router_model`, so the auxiliary model is whatever
+`OLLAMA_INTENT_MODEL` renders to. These tests pin that behaviour end to end.
+"""
+
+import json
+import string
+from pathlib import Path
+
+import pytest
+
+from jarvis.reply.engine import resolve_tool_router_model
+
+TEMPLATE = Path(__file__).resolve().parent.parent / "docker" / "jarvis-config.template.json"
+
+
+def _render(**env) -> dict:
+    raw = TEMPLATE.read_text(encoding="utf-8")
+    return json.loads(string.Template(raw).safe_substitute(**env))
+
+
+class _Cfg:
+    """cfg stand-in carrying only the fields the resolver reads. The template
+    does not render `tool_router_model`, so it stays empty here too."""
+
+    def __init__(self, rendered: dict):
+        self.tool_router_model = rendered.get("tool_router_model", "") or ""
+        self.intent_judge_model = rendered.get("intent_judge_model", "") or ""
+        self.ollama_chat_model = rendered.get("ollama_chat_model", "") or ""
+
+
+def test_template_renders_separate_intent_model():
+    cfg = _render(OLLAMA_CHAT_MODEL="qwen3:8b", OLLAMA_INTENT_MODEL="qwen2.5:3b")
+    assert cfg["ollama_chat_model"] == "qwen3:8b"
+    assert cfg["intent_judge_model"] == "qwen2.5:3b"
+    assert cfg["intent_judge_model"] != cfg["ollama_chat_model"]
+
+
+@pytest.mark.unit
+def test_aux_calls_route_to_small_model_not_chat_model():
+    """The whole point: with a big chat model and a small intent model, tool
+    routing must resolve to the small model, leaving the big model for answers."""
+    cfg = _Cfg(_render(OLLAMA_CHAT_MODEL="qwen3:8b", OLLAMA_INTENT_MODEL="qwen2.5:3b"))
+    assert resolve_tool_router_model(cfg) == "qwen2.5:3b"
+
+
+@pytest.mark.unit
+def test_folding_intent_onto_chat_model_keeps_one_model():
+    """Setting OLLAMA_INTENT_MODEL == OLLAMA_CHAT_MODEL folds everything back
+    onto a single resident model (the documented VRAM-saving opt-out)."""
+    cfg = _Cfg(_render(OLLAMA_CHAT_MODEL="qwen2.5:3b", OLLAMA_INTENT_MODEL="qwen2.5:3b"))
+    assert resolve_tool_router_model(cfg) == "qwen2.5:3b"
+    assert cfg.intent_judge_model == cfg.ollama_chat_model

tests/test_ollama_num_predict.py

@@ -0,0 +1,112 @@
+"""Tests for the ``ollama_num_predict`` chat-output cap.
+
+The cap bounds worst-case reply latency by limiting how many tokens the chat
+model may generate per turn. Spoken (TTS) answers are 1-2 sentences, so the
+default headroom never clips a normal answer and stays above tool-call JSON.
+
+These tests verify behaviour:
+- the config default is present,
+- the value is threaded into the Ollama request as the ``num_predict`` option,
+- the reply loop forwards it to the chat call (and disables it at 0).
+"""
+
+from unittest.mock import Mock, patch
+
+import pytest
+
+from src.jarvis.config import get_default_config
+from src.jarvis.memory.conversation import DialogueMemory
+from src.jarvis.reply.engine import run_reply_engine
+
+
+# ---------------------------------------------------------------------------
+# Config default
+# ---------------------------------------------------------------------------
+
+def test_default_config_has_num_predict_cap():
+    config = get_default_config()
+    assert config["ollama_num_predict"] == 512
+
+
+# ---------------------------------------------------------------------------
+# Transport: extra_options.num_predict reaches the Ollama payload options
+# ---------------------------------------------------------------------------
+
+@patch("jarvis.llm.requests.post")
+def test_chat_with_messages_forwards_num_predict(mock_post):
+    from jarvis.llm import chat_with_messages
+
+    mock_resp = Mock()
+    mock_resp.status_code = 200
+    mock_resp.json.return_value = {"message": {"content": "ok"}}
+    mock_resp.raise_for_status = Mock()
+    mock_post.return_value = mock_resp
+
+    chat_with_messages(
+        "http://localhost:11434",
+        "test-large",
+        [{"role": "user", "content": "hi"}],
+        extra_options={"num_predict": 512},
+    )
+    _, kwargs = mock_post.call_args
+    options = (kwargs.get("json") or {}).get("options") or {}
+    assert options.get("num_predict") == 512
+
+
+# ---------------------------------------------------------------------------
+# Reply loop wiring
+# ---------------------------------------------------------------------------
+
+def _mock_cfg(num_predict):
+    cfg = Mock()
+    cfg.ollama_base_url = "http://localhost:11434"
+    cfg.ollama_chat_model = "test-large"  # avoid SMALL-model text-tool path
+    cfg.ollama_num_predict = num_predict
+    cfg.voice_debug = False
+    cfg.llm_tools_timeout_sec = 8.0
+    cfg.llm_embed_timeout_sec = 10.0
+    cfg.llm_chat_timeout_sec = 45.0
+    cfg.llm_digest_timeout_sec = 8.0
+    cfg.memory_enrichment_max_results = 5
+    cfg.memory_enrichment_source = "diary"
+    cfg.memory_digest_enabled = False
+    cfg.tool_result_digest_enabled = False
+    cfg.location_ip_address = None
+    cfg.location_auto_detect = False
+    cfg.location_enabled = False
+    cfg.agentic_max_turns = 8
+    cfg.tool_search_max_calls = 3
+    cfg.tool_selection_strategy = "all"
+    cfg.tool_carryover_max_turns = 2
+    cfg.tool_carryover_per_entry_chars = 1200
+    cfg.mcps = {}
+    cfg.llm_thinking_enabled = False
+    cfg.tts_engine = "none"
+    cfg.ollama_embed_model = "test-embed"
+    return cfg
+
+
+def _run_single_turn(cfg):
+    """Drive one reply turn that answers in plain text and capture the
+    chat call's extra_options."""
+    with patch("src.jarvis.reply.engine.plan_query", return_value=[]), \
+         patch("src.jarvis.reply.engine.extract_search_params_for_memory", return_value={}), \
+         patch("src.jarvis.reply.engine.extract_text_from_response", return_value="Hello."), \
+         patch("src.jarvis.reply.engine.chat_with_messages") as mock_chat:
+        mock_chat.return_value = {"message": {"content": "Hello."}}
+        run_reply_engine(db=Mock(), cfg=cfg, tts=None,
+                         text="hi", dialogue_memory=DialogueMemory())
+        assert mock_chat.called
+        return mock_chat.call_args.kwargs.get("extra_options")
+
+
+@pytest.mark.unit
+def test_reply_loop_caps_output_when_enabled():
+    extra = _run_single_turn(_mock_cfg(512))
+    assert extra == {"num_predict": 512}
+
+
+@pytest.mark.unit
+def test_reply_loop_no_cap_when_zero():
+    extra = _run_single_turn(_mock_cfg(0))
+    assert extra is None