perf: cap chat output tokens via ollama_num_predict to bound reply latency

Spoken (TTS) replies are 1-2 sentences, so an unbounded num_predict only
exposes the worst case where the chat model rambles or loops. Add an
ollama_num_predict config (default 512, 0 disables) wired into the reply
loop's chat call on both the native- and text-tool paths. The 512-token
headroom stays well above this app's short tool-call JSON, so capping never
truncates a tool call. This keeps the user's quality model instead of
downgrading it. Configurable in the container via OLLAMA_NUM_PREDICT.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

docker/entrypoint.sh

@@ -16,6 +16,9 @@ set -euo pipefail
 # by default so everything runs on one resident model; override if you pull a
 # dedicated small model.
 : "${OLLAMA_INTENT_MODEL:=${OLLAMA_CHAT_MODEL}}"
+# Cap chat-model output tokens per turn (worst-case latency guard). Spoken
+# answers are 1-2 sentences; 512 is safe headroom above tool-call JSON. 0 = off.
+: "${OLLAMA_NUM_PREDICT:=512}"
 : "${OLLAMA_EMBED_MODEL:=nomic-embed-text}"
 : "${WHISPER_MODEL:=small}"
 : "${WHISPER_DEVICE:=cuda}"
@@ -32,7 +35,7 @@ set -euo pipefail
 : "${XDG_RUNTIME_DIR:=/run/user/0}"
 : "${PULSE_SERVER:=unix:${XDG_RUNTIME_DIR}/pulse/native}"
 
-export VNC_RESOLUTION OLLAMA_BASE_URL OLLAMA_CHAT_MODEL OLLAMA_INTENT_MODEL OLLAMA_EMBED_MODEL \
+export VNC_RESOLUTION OLLAMA_BASE_URL OLLAMA_CHAT_MODEL OLLAMA_NUM_PREDICT OLLAMA_INTENT_MODEL OLLAMA_EMBED_MODEL \
        WHISPER_MODEL WHISPER_DEVICE WHISPER_COMPUTE_TYPE JARVIS_DB_PATH \
        PIPER_VOICE PIPER_VOICE_DIR TTS_PIPER_MODEL_PATH BRIDGE_HOST BRIDGE_PORT \
        XDG_RUNTIME_DIR PULSE_SERVER

docker/jarvis-config.template.json

@@ -4,6 +4,7 @@
   "ollama_base_url": "${OLLAMA_BASE_URL}",
   "ollama_embed_model": "${OLLAMA_EMBED_MODEL}",
   "ollama_chat_model": "${OLLAMA_CHAT_MODEL}",
+  "ollama_num_predict": "${OLLAMA_NUM_PREDICT}",
   "intent_judge_model": "${OLLAMA_INTENT_MODEL}",
   "tts_enabled": true,
   "tts_engine": "${TTS_ENGINE}",

docs/llm_contexts.md

@@ -20,7 +20,7 @@ Every distinct LLM call in Jarvis, what feeds it, what consumes it, and how it i
   - Tool schema: native via `generate_tools_json_schema()` ([src/jarvis/tools/registry.py](src/jarvis/tools/registry.py)) or text fallback via `_text_tool_call_guidance()` ([engine.py:68](src/jarvis/reply/engine.py:68))
   - Tool results from prior turns (raw or digested — see #5)
 - **Output**: OpenAI-style `{content, tool_calls, thinking}`. Consumed by the tool orchestrator and TTS pipeline. Natural-language content is delivered immediately; no post-turn evaluator runs.
-- **Limits**: `num_ctx: 8192` (explicit). Timeout `llm_chat_timeout_sec` (45s). Auto-fallback from native to text tool-calls on HTTP 400 (`ToolsNotSupportedError`), sticky for the session. Risk: `fetch_web_page` truncates at 50,000 chars (~37k tokens) — mitigated for SMALL models by tool-result digest (#5) which compresses the payload before it enters the messages history. LARGE models receive the raw payload and may silently see a truncated context.
+- **Limits**: `num_ctx: 8192` (explicit). Output `num_predict: cfg.ollama_num_predict` (default 512, `0`/negative disables) caps generated tokens per turn — a worst-case latency guard for short spoken answers; the headroom stays above tool-call JSON so it does not truncate tool calls (both native and text tool-call paths). Timeout `llm_chat_timeout_sec` (45s). Auto-fallback from native to text tool-calls on HTTP 400 (`ToolsNotSupportedError`), sticky for the session. Risk: `fetch_web_page` truncates at 50,000 chars (~37k tokens) — mitigated for SMALL models by tool-result digest (#5) which compresses the payload before it enters the messages history. LARGE models receive the raw payload and may silently see a truncated context.
 
 ## 2. Intent Judge
 

src/jarvis/config.py

@@ -85,6 +85,12 @@ class Settings:
     llm_digest_timeout_sec: float
     llm_embedding_timeout_sec: float
     llm_profile_select_timeout_sec: float
+    # Upper bound on tokens the chat model may generate per reply turn. Spoken
+    # (TTS) answers are 1-2 sentences, so a cap bounds the worst-case latency of
+    # a model that occasionally rambles or loops without changing normal answers.
+    # The headroom (default 512) sits well above this app's short tool-call JSON,
+    # so capping never truncates a tool call. 0 (or negative) disables the cap.
+    ollama_num_predict: int
 
     # Profiles & Behavior
     active_profiles: list[str]
@@ -394,6 +400,9 @@ def get_default_config() -> Dict[str, Any]:
         "llm_digest_timeout_sec": 8.0,
         "llm_embedding_timeout_sec": 60.0,
         "llm_profile_select_timeout_sec": 30.0,
+        # Cap on chat-model output tokens per turn (worst-case latency guard).
+        # 512 is safe headroom above short TTS answers and tool-call JSON; 0 off.
+        "ollama_num_predict": 512,
 
         # Profiles & Behavior
         "active_profiles": ["developer", "business", "life"],
@@ -763,6 +772,10 @@ def load_settings() -> Settings:
     llm_digest_timeout_sec = float(merged.get("llm_digest_timeout_sec", 8.0))
     llm_embedding_timeout_sec = float(merged.get("llm_embedding_timeout_sec", 60.0))
     llm_profile_select_timeout_sec = float(merged.get("llm_profile_select_timeout_sec", 30.0))
+    try:
+        ollama_num_predict = int(merged.get("ollama_num_predict", 512))
+    except (TypeError, ValueError):
+        ollama_num_predict = 512
 
     return Settings(
         # Database & Storage
@@ -778,6 +791,7 @@ def load_settings() -> Settings:
         llm_digest_timeout_sec=llm_digest_timeout_sec,
         llm_embedding_timeout_sec=llm_embedding_timeout_sec,
         llm_profile_select_timeout_sec=llm_profile_select_timeout_sec,
+        ollama_num_predict=ollama_num_predict,
 
         # Profiles & Behavior
         active_profiles=active_profiles,

src/jarvis/reply/engine.py

@@ -2233,6 +2233,16 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
                 has_tool_calls = " (has tool_calls)" if msg.get("tool_calls") else ""
                 debug_log(f"    [{i}] {role}: {content}{has_tool_calls}", "planning")
 
+        # Bound worst-case generation latency: spoken answers are 1-2 sentences,
+        # so cap the chat model's output tokens. The default headroom sits well
+        # above this app's tool-call JSON, so capping never truncates a tool
+        # call. 0/negative disables it. See config.ollama_num_predict.
+        try:
+            _num_predict = int(getattr(cfg, 'ollama_num_predict', 0) or 0)
+        except (TypeError, ValueError):
+            _num_predict = 0
+        _chat_extra_options = {"num_predict": _num_predict} if _num_predict > 0 else None
+
         # Send messages to Ollama — try native tool calling first, fall back to text-based
         # if the model returns HTTP 400 (native tools API not supported).
         _dump_tools_schema = None if use_text_tools else tools_json_schema
@@ -2242,7 +2252,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
                 chat_model=cfg.ollama_chat_model,
                 messages=messages,
                 timeout_sec=float(getattr(cfg, 'llm_chat_timeout_sec', 45.0)),
-                extra_options=None,
+                extra_options=_chat_extra_options,
                 tools=_dump_tools_schema,
                 thinking=getattr(cfg, 'llm_thinking_enabled', False),
             )
@@ -2273,7 +2283,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
                 chat_model=cfg.ollama_chat_model,
                 messages=messages,
                 timeout_sec=float(getattr(cfg, 'llm_chat_timeout_sec', 45.0)),
-                extra_options=None,
+                extra_options=_chat_extra_options,
                 tools=None,
                 thinking=getattr(cfg, 'llm_thinking_enabled', False),
             )

src/jarvis/reply/reply.spec.md

@@ -287,6 +287,8 @@ Turn 4: LLM → {content: "Here's a comprehensive comparison of the iPhone 15 mo
   - `llm_tools_timeout_sec` (enrichment extraction)
   - `llm_embed_timeout_sec` (vector search)
   - `llm_chat_timeout_sec` (messages loop turn)
+- Output bound:
+  - `ollama_num_predict` (default `512`, `0`/negative disables) caps the chat model's generated tokens per turn via the Ollama `num_predict` option on the messages-loop call. Spoken (TTS) answers are 1-2 sentences, so this never clips a normal answer; it bounds the worst-case latency of a model that occasionally rambles or loops. The default headroom sits well above this app's short tool-call JSON, so it does not truncate tool calls. Applied uniformly to the reply loop's chat call (both native-tools and text-tools paths); the small classification passes (intent judge, digests) keep their own caps. Note: this is a worst-case guard, not the primary latency lever, which is model size and GPU residency.
 - Memory enrichment:
   - `memory_enrichment_max_results` limits recalled snippets.
   - `memory_digest_enabled` (default `null` = auto-on for SMALL models ≤7B, off for LARGE) distils the combined diary + graph dump into a short relevance-filtered note via a cheap LLM pass before injecting into the system prompt. See **Memory Digest for Small Models** below.

tests/test_ollama_num_predict.py

@@ -0,0 +1,112 @@
+"""Tests for the ``ollama_num_predict`` chat-output cap.
+
+The cap bounds worst-case reply latency by limiting how many tokens the chat
+model may generate per turn. Spoken (TTS) answers are 1-2 sentences, so the
+default headroom never clips a normal answer and stays above tool-call JSON.
+
+These tests verify behaviour:
+- the config default is present,
+- the value is threaded into the Ollama request as the ``num_predict`` option,
+- the reply loop forwards it to the chat call (and disables it at 0).
+"""
+
+from unittest.mock import Mock, patch
+
+import pytest
+
+from src.jarvis.config import get_default_config
+from src.jarvis.memory.conversation import DialogueMemory
+from src.jarvis.reply.engine import run_reply_engine
+
+
+# ---------------------------------------------------------------------------
+# Config default
+# ---------------------------------------------------------------------------
+
+def test_default_config_has_num_predict_cap():
+    config = get_default_config()
+    assert config["ollama_num_predict"] == 512
+
+
+# ---------------------------------------------------------------------------
+# Transport: extra_options.num_predict reaches the Ollama payload options
+# ---------------------------------------------------------------------------
+
+@patch("jarvis.llm.requests.post")
+def test_chat_with_messages_forwards_num_predict(mock_post):
+    from jarvis.llm import chat_with_messages
+
+    mock_resp = Mock()
+    mock_resp.status_code = 200
+    mock_resp.json.return_value = {"message": {"content": "ok"}}
+    mock_resp.raise_for_status = Mock()
+    mock_post.return_value = mock_resp
+
+    chat_with_messages(
+        "http://localhost:11434",
+        "test-large",
+        [{"role": "user", "content": "hi"}],
+        extra_options={"num_predict": 512},
+    )
+    _, kwargs = mock_post.call_args
+    options = (kwargs.get("json") or {}).get("options") or {}
+    assert options.get("num_predict") == 512
+
+
+# ---------------------------------------------------------------------------
+# Reply loop wiring
+# ---------------------------------------------------------------------------
+
+def _mock_cfg(num_predict):
+    cfg = Mock()
+    cfg.ollama_base_url = "http://localhost:11434"
+    cfg.ollama_chat_model = "test-large"  # avoid SMALL-model text-tool path
+    cfg.ollama_num_predict = num_predict
+    cfg.voice_debug = False
+    cfg.llm_tools_timeout_sec = 8.0
+    cfg.llm_embed_timeout_sec = 10.0
+    cfg.llm_chat_timeout_sec = 45.0
+    cfg.llm_digest_timeout_sec = 8.0
+    cfg.memory_enrichment_max_results = 5
+    cfg.memory_enrichment_source = "diary"
+    cfg.memory_digest_enabled = False
+    cfg.tool_result_digest_enabled = False
+    cfg.location_ip_address = None
+    cfg.location_auto_detect = False
+    cfg.location_enabled = False
+    cfg.agentic_max_turns = 8
+    cfg.tool_search_max_calls = 3
+    cfg.tool_selection_strategy = "all"
+    cfg.tool_carryover_max_turns = 2
+    cfg.tool_carryover_per_entry_chars = 1200
+    cfg.mcps = {}
+    cfg.llm_thinking_enabled = False
+    cfg.tts_engine = "none"
+    cfg.ollama_embed_model = "test-embed"
+    return cfg
+
+
+def _run_single_turn(cfg):
+    """Drive one reply turn that answers in plain text and capture the
+    chat call's extra_options."""
+    with patch("src.jarvis.reply.engine.plan_query", return_value=[]), \
+         patch("src.jarvis.reply.engine.extract_search_params_for_memory", return_value={}), \
+         patch("src.jarvis.reply.engine.extract_text_from_response", return_value="Hello."), \
+         patch("src.jarvis.reply.engine.chat_with_messages") as mock_chat:
+        mock_chat.return_value = {"message": {"content": "Hello."}}
+        run_reply_engine(db=Mock(), cfg=cfg, tts=None,
+                         text="hi", dialogue_memory=DialogueMemory())
+        assert mock_chat.called
+        return mock_chat.call_args.kwargs.get("extra_options")
+
+
+@pytest.mark.unit
+def test_reply_loop_caps_output_when_enabled():
+    extra = _run_single_turn(_mock_cfg(512))
+    assert extra == {"num_predict": 512}
+
+
+@pytest.mark.unit
+def test_reply_loop_no_cap_when_zero():
+    extra = _run_single_turn(_mock_cfg(0))
+    assert extra is None