diff --git a/.env.example b/.env.example index 8ac1a11..7ee097d 100644 --- a/.env.example +++ b/.env.example @@ -47,8 +47,11 @@ MELO_SPEED=1.5 # --------------------------------------------------------------------------- # In docker-compose this is overridden to http://ollama:11434 automatically. OLLAMA_BASE_URL=http://127.0.0.1:11434 -# qwen3:8b — best 8GB-VRAM pick: strongest tool-calling, ~5GB Q4, fits the RTX 5050. -OLLAMA_CHAT_MODEL=qwen3:8b +# qwen2.5:3b — small non-reasoning instruct model. ~2.4GB, runs 100% on the GPU +# (the 8B offloads ~8% to CPU), warm voice turns ~2-4s vs ~5-7s on 8B. Clean +# Korean on factual/tool replies; can occasionally leak a trailing CJK phrase on +# free-form chit-chat. Swap back to qwen3:8b for the strongest tool-calling. +OLLAMA_CHAT_MODEL=qwen2.5:3b OLLAMA_EMBED_MODEL=nomic-embed-text WHISPER_MODEL=small diff --git a/docker-compose.yml b/docker-compose.yml index 5e69d72..9722317 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,11 +17,12 @@ services: ollama: image: ollama/ollama:latest restart: unless-stopped - environment: - # Keep the chat + embed models resident so voice turns never pay a cold - # reload. Default keep_alive is 5 min, so every post-idle turn took - # ~30-60s while Qwen3 8B reloaded into the GPU. -1 = never unload. - OLLAMA_KEEP_ALIVE: "-1" + # Model residency is controlled per-request, not globally. The brain pins + # the chat model with keep_alive=30m (src/jarvis/llm.py) so voice turns + # never pay a cold reload, while embeddings pass keep_alive=0 + # (src/jarvis/memory/embeddings.py) so nomic-embed unloads right after use. + # A global OLLAMA_KEEP_ALIVE=-1 was removed because it also kept the embed + # model resident forever, wasting VRAM next to the chat model. volumes: - ollama_models:/root/.ollama # GPU: needs nvidia-container-toolkit on the host (CDI). Verified on the @@ -37,7 +38,7 @@ services: restart: "no" environment: OLLAMA_HOST: http://ollama:11434 - CHAT_MODEL: ${OLLAMA_CHAT_MODEL:-qwen3:8b} + CHAT_MODEL: ${OLLAMA_CHAT_MODEL:-qwen2.5:3b} EMBED_MODEL: ${OLLAMA_EMBED_MODEL:-nomic-embed-text} entrypoint: ["/bin/sh", "-c"] command: @@ -59,7 +60,7 @@ services: environment: # Point the brain at the ollama service and the bot at the in-container bridge. OLLAMA_BASE_URL: http://ollama:11434 - OLLAMA_CHAT_MODEL: ${OLLAMA_CHAT_MODEL:-qwen3:8b} + OLLAMA_CHAT_MODEL: ${OLLAMA_CHAT_MODEL:-qwen2.5:3b} OLLAMA_EMBED_MODEL: ${OLLAMA_EMBED_MODEL:-nomic-embed-text} WHISPER_MODEL: ${WHISPER_MODEL:-small} WHISPER_DEVICE: ${WHISPER_DEVICE:-cuda} diff --git a/src/jarvis/llm.py b/src/jarvis/llm.py index a32865a..c3d0fff 100644 --- a/src/jarvis/llm.py +++ b/src/jarvis/llm.py @@ -41,8 +41,14 @@ def call_llm_direct(base_url: str, chat_model: str, system_prompt: str, user_con "stream": False, "options": options, "think": thinking, + # Keep the chat model resident between calls. Without an explicit + # keep_alive Ollama evicts it after its default idle window and the + # next turn pays a cold reload. We pin the chat model only (embeddings + # pass keep_alive=0 so they unload after use) instead of a global + # OLLAMA_KEEP_ALIVE=-1, which would keep every model resident forever. + "keep_alive": "30m", } - + try: with requests.post(f"{base_url.rstrip('/')}/api/chat", json=payload, timeout=timeout_sec) as resp: resp.raise_for_status() @@ -98,6 +104,8 @@ def call_llm_streaming( "stream": True, "options": {"num_ctx": 4096}, "think": thinking, + # Keep the chat model resident between calls (see call_llm_direct). + "keep_alive": "30m", } # Use ``with`` so the streaming response (and the underlying TCP @@ -201,6 +209,8 @@ def chat_with_messages( "stream": False, "options": {"num_ctx": 8192}, "think": thinking, + # Keep the chat model resident between turns (see call_llm_direct). + "keep_alive": "30m", } if extra_options and isinstance(extra_options, dict): # Merge shallowly into options diff --git a/src/jarvis/memory/embeddings.py b/src/jarvis/memory/embeddings.py index d6dc524..9ad20c5 100644 --- a/src/jarvis/memory/embeddings.py +++ b/src/jarvis/memory/embeddings.py @@ -6,7 +6,11 @@ def get_embedding(text: str, base_url: str, model: str, timeout_sec: float = 15. try: resp = requests.post( f"{base_url.rstrip('/')}/api/embeddings", - json={"model": model, "prompt": text}, + # keep_alive=0 unloads the embedding model right after the call so + # it does not sit resident in VRAM alongside the chat model. The + # chat model is pinned separately (llm.py keep_alive=30m); only the + # actively-used chat model should stay loaded. + json={"model": model, "prompt": text, "keep_alive": 0}, timeout=timeout_sec, ) resp.raise_for_status()