diff --git a/src/jarvis/llm.py b/src/jarvis/llm.py index c3d0fff..046be58 100644 --- a/src/jarvis/llm.py +++ b/src/jarvis/llm.py @@ -2,24 +2,37 @@ from __future__ import annotations from typing import Optional, Any, Dict, List, Generator, Callable +import os import requests import json from .debug import debug_log +# Single context-window size shared by EVERY Ollama call (chat, router, +# enrichment, digests, streaming). Ollama keeps a SEPARATE loaded model +# instance per (model, num_ctx): mixing 4096 and 8192 in one voice turn made +# it evict and cold-reload the model (~3.4s each) on every context switch — +# the dominant per-turn latency. Keeping one value collapses this to a single +# resident instance, so only the very first call of a cold model pays a load. +# 8192 is the floor the main agentic chat needs (system prompt + tool schema + +# memory) without silent truncation. Tunable via env for tight-VRAM hosts. +OLLAMA_NUM_CTX = int(os.environ.get("OLLAMA_NUM_CTX", "8192")) + + class ToolsNotSupportedError(Exception): """Raised when the model returns HTTP 400 because native tool calling is not supported.""" pass -def call_llm_direct(base_url: str, chat_model: str, system_prompt: str, user_content: str, timeout_sec: float = 10.0, thinking: bool = False, num_ctx: int = 4096, temperature: Optional[float] = None) -> Optional[str]: +def call_llm_direct(base_url: str, chat_model: str, system_prompt: str, user_content: str, timeout_sec: float = 10.0, thinking: bool = False, num_ctx: int = OLLAMA_NUM_CTX, temperature: Optional[float] = None) -> Optional[str]: """Direct LLM call without temporal context, location, or other ask_coach features. - ``num_ctx`` controls Ollama's context window for this call. Default 4096 is - fine for small classification-shaped passes; callers that assemble richer - prompts (planner with dialogue + memory + tool catalogue) should pass a - larger value to avoid silent truncation. + ``num_ctx`` controls Ollama's context window for this call. It defaults to + the shared ``OLLAMA_NUM_CTX`` so small classification-shaped passes load the + SAME Ollama instance as the main chat loop (no cold reload on context + switch). Callers may still override it, but diverging from the shared value + reintroduces a per-turn model reload. ``temperature`` is forwarded to Ollama when set. Pass ``0.0`` for classification / extraction calls where determinism beats creativity — @@ -102,7 +115,7 @@ def call_llm_streaming( "model": chat_model, "messages": messages, "stream": True, - "options": {"num_ctx": 4096}, + "options": {"num_ctx": OLLAMA_NUM_CTX}, "think": thinking, # Keep the chat model resident between calls (see call_llm_direct). "keep_alive": "30m", @@ -207,7 +220,7 @@ def chat_with_messages( "model": chat_model, "messages": messages, "stream": False, - "options": {"num_ctx": 8192}, + "options": {"num_ctx": OLLAMA_NUM_CTX}, "think": thinking, # Keep the chat model resident between turns (see call_llm_direct). "keep_alive": "30m", diff --git a/src/jarvis/reply/planner.py b/src/jarvis/reply/planner.py index 327ad6b..ea0eea6 100644 --- a/src/jarvis/reply/planner.py +++ b/src/jarvis/reply/planner.py @@ -40,7 +40,7 @@ import re from typing import List, Optional, Sequence, Tuple from ..debug import debug_log -from ..llm import call_llm_direct +from ..llm import call_llm_direct, OLLAMA_NUM_CTX # Hard cap on plan length. Small models happily emit 10+ step plans that @@ -441,7 +441,7 @@ def plan_query( user_content=user_content, timeout_sec=effective_timeout, thinking=False, - num_ctx=8192, + num_ctx=OLLAMA_NUM_CTX, ) except Exception as exc: # pragma: no cover — defensive debug_log(f"planner: LLM call failed — {exc}", "planning") @@ -716,7 +716,7 @@ def resolve_next_tool_call( user_content=user_content, timeout_sec=effective_timeout, thinking=False, - num_ctx=8192, + num_ctx=OLLAMA_NUM_CTX, ) except Exception as exc: # pragma: no cover — defensive debug_log(f"planner.resolve_next_tool_call: LLM failed — {exc}", "planning")