diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 22bab02..23bbabc 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -16,6 +16,9 @@ set -euo pipefail # by default so everything runs on one resident model; override if you pull a # dedicated small model. : "${OLLAMA_INTENT_MODEL:=${OLLAMA_CHAT_MODEL}}" +# Cap chat-model output tokens per turn (worst-case latency guard). Spoken +# answers are 1-2 sentences; 512 is safe headroom above tool-call JSON. 0 = off. +: "${OLLAMA_NUM_PREDICT:=512}" : "${OLLAMA_EMBED_MODEL:=nomic-embed-text}" : "${WHISPER_MODEL:=small}" : "${WHISPER_DEVICE:=cuda}" @@ -32,7 +35,7 @@ set -euo pipefail : "${XDG_RUNTIME_DIR:=/run/user/0}" : "${PULSE_SERVER:=unix:${XDG_RUNTIME_DIR}/pulse/native}" -export VNC_RESOLUTION OLLAMA_BASE_URL OLLAMA_CHAT_MODEL OLLAMA_INTENT_MODEL OLLAMA_EMBED_MODEL \ +export VNC_RESOLUTION OLLAMA_BASE_URL OLLAMA_CHAT_MODEL OLLAMA_NUM_PREDICT OLLAMA_INTENT_MODEL OLLAMA_EMBED_MODEL \ WHISPER_MODEL WHISPER_DEVICE WHISPER_COMPUTE_TYPE JARVIS_DB_PATH \ PIPER_VOICE PIPER_VOICE_DIR TTS_PIPER_MODEL_PATH BRIDGE_HOST BRIDGE_PORT \ XDG_RUNTIME_DIR PULSE_SERVER diff --git a/docker/jarvis-config.template.json b/docker/jarvis-config.template.json index dde5c3c..bf72626 100644 --- a/docker/jarvis-config.template.json +++ b/docker/jarvis-config.template.json @@ -4,6 +4,7 @@ "ollama_base_url": "${OLLAMA_BASE_URL}", "ollama_embed_model": "${OLLAMA_EMBED_MODEL}", "ollama_chat_model": "${OLLAMA_CHAT_MODEL}", + "ollama_num_predict": "${OLLAMA_NUM_PREDICT}", "intent_judge_model": "${OLLAMA_INTENT_MODEL}", "tts_enabled": true, "tts_engine": "${TTS_ENGINE}", diff --git a/docs/llm_contexts.md b/docs/llm_contexts.md index f9ffb09..f16ab45 100644 --- a/docs/llm_contexts.md +++ b/docs/llm_contexts.md @@ -20,7 +20,7 @@ Every distinct LLM call in Jarvis, what feeds it, what consumes it, and how it i - Tool schema: native via `generate_tools_json_schema()` ([src/jarvis/tools/registry.py](src/jarvis/tools/registry.py)) or text fallback via `_text_tool_call_guidance()` ([engine.py:68](src/jarvis/reply/engine.py:68)) - Tool results from prior turns (raw or digested — see #5) - **Output**: OpenAI-style `{content, tool_calls, thinking}`. Consumed by the tool orchestrator and TTS pipeline. Natural-language content is delivered immediately; no post-turn evaluator runs. -- **Limits**: `num_ctx: 8192` (explicit). Timeout `llm_chat_timeout_sec` (45s). Auto-fallback from native to text tool-calls on HTTP 400 (`ToolsNotSupportedError`), sticky for the session. Risk: `fetch_web_page` truncates at 50,000 chars (~37k tokens) — mitigated for SMALL models by tool-result digest (#5) which compresses the payload before it enters the messages history. LARGE models receive the raw payload and may silently see a truncated context. +- **Limits**: `num_ctx: 8192` (explicit). Output `num_predict: cfg.ollama_num_predict` (default 512, `0`/negative disables) caps generated tokens per turn — a worst-case latency guard for short spoken answers; the headroom stays above tool-call JSON so it does not truncate tool calls (both native and text tool-call paths). Timeout `llm_chat_timeout_sec` (45s). Auto-fallback from native to text tool-calls on HTTP 400 (`ToolsNotSupportedError`), sticky for the session. Risk: `fetch_web_page` truncates at 50,000 chars (~37k tokens) — mitigated for SMALL models by tool-result digest (#5) which compresses the payload before it enters the messages history. LARGE models receive the raw payload and may silently see a truncated context. ## 2. Intent Judge diff --git a/src/jarvis/config.py b/src/jarvis/config.py index 854861f..3e5233a 100644 --- a/src/jarvis/config.py +++ b/src/jarvis/config.py @@ -85,6 +85,12 @@ class Settings: llm_digest_timeout_sec: float llm_embedding_timeout_sec: float llm_profile_select_timeout_sec: float + # Upper bound on tokens the chat model may generate per reply turn. Spoken + # (TTS) answers are 1-2 sentences, so a cap bounds the worst-case latency of + # a model that occasionally rambles or loops without changing normal answers. + # The headroom (default 512) sits well above this app's short tool-call JSON, + # so capping never truncates a tool call. 0 (or negative) disables the cap. + ollama_num_predict: int # Profiles & Behavior active_profiles: list[str] @@ -394,6 +400,9 @@ def get_default_config() -> Dict[str, Any]: "llm_digest_timeout_sec": 8.0, "llm_embedding_timeout_sec": 60.0, "llm_profile_select_timeout_sec": 30.0, + # Cap on chat-model output tokens per turn (worst-case latency guard). + # 512 is safe headroom above short TTS answers and tool-call JSON; 0 off. + "ollama_num_predict": 512, # Profiles & Behavior "active_profiles": ["developer", "business", "life"], @@ -763,6 +772,10 @@ def load_settings() -> Settings: llm_digest_timeout_sec = float(merged.get("llm_digest_timeout_sec", 8.0)) llm_embedding_timeout_sec = float(merged.get("llm_embedding_timeout_sec", 60.0)) llm_profile_select_timeout_sec = float(merged.get("llm_profile_select_timeout_sec", 30.0)) + try: + ollama_num_predict = int(merged.get("ollama_num_predict", 512)) + except (TypeError, ValueError): + ollama_num_predict = 512 return Settings( # Database & Storage @@ -778,6 +791,7 @@ def load_settings() -> Settings: llm_digest_timeout_sec=llm_digest_timeout_sec, llm_embedding_timeout_sec=llm_embedding_timeout_sec, llm_profile_select_timeout_sec=llm_profile_select_timeout_sec, + ollama_num_predict=ollama_num_predict, # Profiles & Behavior active_profiles=active_profiles, diff --git a/src/jarvis/reply/engine.py b/src/jarvis/reply/engine.py index 779cdb2..a0f41b4 100644 --- a/src/jarvis/reply/engine.py +++ b/src/jarvis/reply/engine.py @@ -2233,6 +2233,16 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any], has_tool_calls = " (has tool_calls)" if msg.get("tool_calls") else "" debug_log(f" [{i}] {role}: {content}{has_tool_calls}", "planning") + # Bound worst-case generation latency: spoken answers are 1-2 sentences, + # so cap the chat model's output tokens. The default headroom sits well + # above this app's tool-call JSON, so capping never truncates a tool + # call. 0/negative disables it. See config.ollama_num_predict. + try: + _num_predict = int(getattr(cfg, 'ollama_num_predict', 0) or 0) + except (TypeError, ValueError): + _num_predict = 0 + _chat_extra_options = {"num_predict": _num_predict} if _num_predict > 0 else None + # Send messages to Ollama — try native tool calling first, fall back to text-based # if the model returns HTTP 400 (native tools API not supported). _dump_tools_schema = None if use_text_tools else tools_json_schema @@ -2242,7 +2252,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any], chat_model=cfg.ollama_chat_model, messages=messages, timeout_sec=float(getattr(cfg, 'llm_chat_timeout_sec', 45.0)), - extra_options=None, + extra_options=_chat_extra_options, tools=_dump_tools_schema, thinking=getattr(cfg, 'llm_thinking_enabled', False), ) @@ -2273,7 +2283,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any], chat_model=cfg.ollama_chat_model, messages=messages, timeout_sec=float(getattr(cfg, 'llm_chat_timeout_sec', 45.0)), - extra_options=None, + extra_options=_chat_extra_options, tools=None, thinking=getattr(cfg, 'llm_thinking_enabled', False), ) diff --git a/src/jarvis/reply/reply.spec.md b/src/jarvis/reply/reply.spec.md index 27076a7..2d1ecd1 100644 --- a/src/jarvis/reply/reply.spec.md +++ b/src/jarvis/reply/reply.spec.md @@ -287,6 +287,8 @@ Turn 4: LLM → {content: "Here's a comprehensive comparison of the iPhone 15 mo - `llm_tools_timeout_sec` (enrichment extraction) - `llm_embed_timeout_sec` (vector search) - `llm_chat_timeout_sec` (messages loop turn) +- Output bound: + - `ollama_num_predict` (default `512`, `0`/negative disables) caps the chat model's generated tokens per turn via the Ollama `num_predict` option on the messages-loop call. Spoken (TTS) answers are 1-2 sentences, so this never clips a normal answer; it bounds the worst-case latency of a model that occasionally rambles or loops. The default headroom sits well above this app's short tool-call JSON, so it does not truncate tool calls. Applied uniformly to the reply loop's chat call (both native-tools and text-tools paths); the small classification passes (intent judge, digests) keep their own caps. Note: this is a worst-case guard, not the primary latency lever, which is model size and GPU residency. - Memory enrichment: - `memory_enrichment_max_results` limits recalled snippets. - `memory_digest_enabled` (default `null` = auto-on for SMALL models ≤7B, off for LARGE) distils the combined diary + graph dump into a short relevance-filtered note via a cheap LLM pass before injecting into the system prompt. See **Memory Digest for Small Models** below. diff --git a/tests/test_ollama_num_predict.py b/tests/test_ollama_num_predict.py new file mode 100644 index 0000000..af259bb --- /dev/null +++ b/tests/test_ollama_num_predict.py @@ -0,0 +1,112 @@ +"""Tests for the ``ollama_num_predict`` chat-output cap. + +The cap bounds worst-case reply latency by limiting how many tokens the chat +model may generate per turn. Spoken (TTS) answers are 1-2 sentences, so the +default headroom never clips a normal answer and stays above tool-call JSON. + +These tests verify behaviour: +- the config default is present, +- the value is threaded into the Ollama request as the ``num_predict`` option, +- the reply loop forwards it to the chat call (and disables it at 0). +""" + +from unittest.mock import Mock, patch + +import pytest + +from src.jarvis.config import get_default_config +from src.jarvis.memory.conversation import DialogueMemory +from src.jarvis.reply.engine import run_reply_engine + + +# --------------------------------------------------------------------------- +# Config default +# --------------------------------------------------------------------------- + +def test_default_config_has_num_predict_cap(): + config = get_default_config() + assert config["ollama_num_predict"] == 512 + + +# --------------------------------------------------------------------------- +# Transport: extra_options.num_predict reaches the Ollama payload options +# --------------------------------------------------------------------------- + +@patch("jarvis.llm.requests.post") +def test_chat_with_messages_forwards_num_predict(mock_post): + from jarvis.llm import chat_with_messages + + mock_resp = Mock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"message": {"content": "ok"}} + mock_resp.raise_for_status = Mock() + mock_post.return_value = mock_resp + + chat_with_messages( + "http://localhost:11434", + "test-large", + [{"role": "user", "content": "hi"}], + extra_options={"num_predict": 512}, + ) + _, kwargs = mock_post.call_args + options = (kwargs.get("json") or {}).get("options") or {} + assert options.get("num_predict") == 512 + + +# --------------------------------------------------------------------------- +# Reply loop wiring +# --------------------------------------------------------------------------- + +def _mock_cfg(num_predict): + cfg = Mock() + cfg.ollama_base_url = "http://localhost:11434" + cfg.ollama_chat_model = "test-large" # avoid SMALL-model text-tool path + cfg.ollama_num_predict = num_predict + cfg.voice_debug = False + cfg.llm_tools_timeout_sec = 8.0 + cfg.llm_embed_timeout_sec = 10.0 + cfg.llm_chat_timeout_sec = 45.0 + cfg.llm_digest_timeout_sec = 8.0 + cfg.memory_enrichment_max_results = 5 + cfg.memory_enrichment_source = "diary" + cfg.memory_digest_enabled = False + cfg.tool_result_digest_enabled = False + cfg.location_ip_address = None + cfg.location_auto_detect = False + cfg.location_enabled = False + cfg.agentic_max_turns = 8 + cfg.tool_search_max_calls = 3 + cfg.tool_selection_strategy = "all" + cfg.tool_carryover_max_turns = 2 + cfg.tool_carryover_per_entry_chars = 1200 + cfg.mcps = {} + cfg.llm_thinking_enabled = False + cfg.tts_engine = "none" + cfg.ollama_embed_model = "test-embed" + return cfg + + +def _run_single_turn(cfg): + """Drive one reply turn that answers in plain text and capture the + chat call's extra_options.""" + with patch("src.jarvis.reply.engine.plan_query", return_value=[]), \ + patch("src.jarvis.reply.engine.extract_search_params_for_memory", return_value={}), \ + patch("src.jarvis.reply.engine.extract_text_from_response", return_value="Hello."), \ + patch("src.jarvis.reply.engine.chat_with_messages") as mock_chat: + mock_chat.return_value = {"message": {"content": "Hello."}} + run_reply_engine(db=Mock(), cfg=cfg, tts=None, + text="hi", dialogue_memory=DialogueMemory()) + assert mock_chat.called + return mock_chat.call_args.kwargs.get("extra_options") + + +@pytest.mark.unit +def test_reply_loop_caps_output_when_enabled(): + extra = _run_single_turn(_mock_cfg(512)) + assert extra == {"num_predict": 512} + + +@pytest.mark.unit +def test_reply_loop_no_cap_when_zero(): + extra = _run_single_turn(_mock_cfg(0)) + assert extra is None