perf: cap chat output tokens via ollama_num_predict to bound reply latency

Spoken (TTS) replies are 1-2 sentences, so an unbounded num_predict only
exposes the worst case where the chat model rambles or loops. Add an
ollama_num_predict config (default 512, 0 disables) wired into the reply
loop's chat call on both the native- and text-tool paths. The 512-token
headroom stays well above this app's short tool-call JSON, so capping never
truncates a tool call. This keeps the user's quality model instead of
downgrading it. Configurable in the container via OLLAMA_NUM_PREDICT.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

src/jarvis/config.py

@@ -85,6 +85,12 @@ class Settings:
     llm_digest_timeout_sec: float
     llm_embedding_timeout_sec: float
     llm_profile_select_timeout_sec: float
+    # Upper bound on tokens the chat model may generate per reply turn. Spoken
+    # (TTS) answers are 1-2 sentences, so a cap bounds the worst-case latency of
+    # a model that occasionally rambles or loops without changing normal answers.
+    # The headroom (default 512) sits well above this app's short tool-call JSON,
+    # so capping never truncates a tool call. 0 (or negative) disables the cap.
+    ollama_num_predict: int
 
     # Profiles & Behavior
     active_profiles: list[str]
@@ -394,6 +400,9 @@ def get_default_config() -> Dict[str, Any]:
         "llm_digest_timeout_sec": 8.0,
         "llm_embedding_timeout_sec": 60.0,
         "llm_profile_select_timeout_sec": 30.0,
+        # Cap on chat-model output tokens per turn (worst-case latency guard).
+        # 512 is safe headroom above short TTS answers and tool-call JSON; 0 off.
+        "ollama_num_predict": 512,
 
         # Profiles & Behavior
         "active_profiles": ["developer", "business", "life"],
@@ -763,6 +772,10 @@ def load_settings() -> Settings:
     llm_digest_timeout_sec = float(merged.get("llm_digest_timeout_sec", 8.0))
     llm_embedding_timeout_sec = float(merged.get("llm_embedding_timeout_sec", 60.0))
     llm_profile_select_timeout_sec = float(merged.get("llm_profile_select_timeout_sec", 30.0))
+    try:
+        ollama_num_predict = int(merged.get("ollama_num_predict", 512))
+    except (TypeError, ValueError):
+        ollama_num_predict = 512
 
     return Settings(
         # Database & Storage
@@ -778,6 +791,7 @@ def load_settings() -> Settings:
         llm_digest_timeout_sec=llm_digest_timeout_sec,
         llm_embedding_timeout_sec=llm_embedding_timeout_sec,
         llm_profile_select_timeout_sec=llm_profile_select_timeout_sec,
+        ollama_num_predict=ollama_num_predict,
 
         # Profiles & Behavior
         active_profiles=active_profiles,

src/jarvis/reply/engine.py

@@ -2233,6 +2233,16 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
                 has_tool_calls = " (has tool_calls)" if msg.get("tool_calls") else ""
                 debug_log(f"    [{i}] {role}: {content}{has_tool_calls}", "planning")
 
+        # Bound worst-case generation latency: spoken answers are 1-2 sentences,
+        # so cap the chat model's output tokens. The default headroom sits well
+        # above this app's tool-call JSON, so capping never truncates a tool
+        # call. 0/negative disables it. See config.ollama_num_predict.
+        try:
+            _num_predict = int(getattr(cfg, 'ollama_num_predict', 0) or 0)
+        except (TypeError, ValueError):
+            _num_predict = 0
+        _chat_extra_options = {"num_predict": _num_predict} if _num_predict > 0 else None
+
         # Send messages to Ollama — try native tool calling first, fall back to text-based
         # if the model returns HTTP 400 (native tools API not supported).
         _dump_tools_schema = None if use_text_tools else tools_json_schema
@@ -2242,7 +2252,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
                 chat_model=cfg.ollama_chat_model,
                 messages=messages,
                 timeout_sec=float(getattr(cfg, 'llm_chat_timeout_sec', 45.0)),
-                extra_options=None,
+                extra_options=_chat_extra_options,
                 tools=_dump_tools_schema,
                 thinking=getattr(cfg, 'llm_thinking_enabled', False),
             )
@@ -2273,7 +2283,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
                 chat_model=cfg.ollama_chat_model,
                 messages=messages,
                 timeout_sec=float(getattr(cfg, 'llm_chat_timeout_sec', 45.0)),
-                extra_options=None,
+                extra_options=_chat_extra_options,
                 tools=None,
                 thinking=getattr(cfg, 'llm_thinking_enabled', False),
             )

src/jarvis/reply/reply.spec.md

@@ -287,6 +287,8 @@ Turn 4: LLM → {content: "Here's a comprehensive comparison of the iPhone 15 mo
   - `llm_tools_timeout_sec` (enrichment extraction)
   - `llm_embed_timeout_sec` (vector search)
   - `llm_chat_timeout_sec` (messages loop turn)
+- Output bound:
+  - `ollama_num_predict` (default `512`, `0`/negative disables) caps the chat model's generated tokens per turn via the Ollama `num_predict` option on the messages-loop call. Spoken (TTS) answers are 1-2 sentences, so this never clips a normal answer; it bounds the worst-case latency of a model that occasionally rambles or loops. The default headroom sits well above this app's short tool-call JSON, so it does not truncate tool calls. Applied uniformly to the reply loop's chat call (both native-tools and text-tools paths); the small classification passes (intent judge, digests) keep their own caps. Note: this is a worst-case guard, not the primary latency lever, which is model size and GPU residency.
 - Memory enrichment:
   - `memory_enrichment_max_results` limits recalled snippets.
   - `memory_digest_enabled` (default `null` = auto-on for SMALL models ≤7B, off for LARGE) distils the combined diary + graph dump into a short relevance-filtered note via a cheap LLM pass before injecting into the system prompt. See **Memory Digest for Small Models** below.