perf(brain): pin chat model per-request, unload embeddings; default qwen2.5:3b
Replace the blunt global OLLAMA_KEEP_ALIVE=-1 (which kept every model, including nomic-embed, resident in VRAM forever) with per-request residency: - llm.py: all three /api/chat payloads send keep_alive=30m so the actively used chat model stays resident and voice turns never pay a cold reload. - embeddings.py: /api/embeddings sends keep_alive=0 so nomic-embed unloads right after each call instead of squatting in VRAM next to the chat model. - docker-compose.yml: drop the global OLLAMA_KEEP_ALIVE=-1; document the per-request scheme on the ollama service. Switch the default chat model qwen3:8b -> qwen2.5:3b. Verified live on the RTX 5050 (8GB): - ollama ps: qwen2.5:3b 2.4GB, 100% GPU (8B was 92% GPU / 8% CPU), UNTIL ~30m (the 30m pin, not "Forever"); nomic-embed absent after several enriched turns. - nvidia-smi: ~3.2GB VRAM used total (qwen 2.4GB + whisper 0.7GB) vs ~6.6GB. - Korean /text turns: warm 1.7-4s (cold first load ~52s), vs ~5-7s on 8B; time/weather/places tool calls fire and reply in Korean. Known limitation: qwen2.5:3b can occasionally leak a trailing CJK phrase on free-form chit-chat (factual/tool replies stay clean).
This commit is contained in:
@@ -41,8 +41,14 @@ def call_llm_direct(base_url: str, chat_model: str, system_prompt: str, user_con
|
||||
"stream": False,
|
||||
"options": options,
|
||||
"think": thinking,
|
||||
# Keep the chat model resident between calls. Without an explicit
|
||||
# keep_alive Ollama evicts it after its default idle window and the
|
||||
# next turn pays a cold reload. We pin the chat model only (embeddings
|
||||
# pass keep_alive=0 so they unload after use) instead of a global
|
||||
# OLLAMA_KEEP_ALIVE=-1, which would keep every model resident forever.
|
||||
"keep_alive": "30m",
|
||||
}
|
||||
|
||||
|
||||
try:
|
||||
with requests.post(f"{base_url.rstrip('/')}/api/chat", json=payload, timeout=timeout_sec) as resp:
|
||||
resp.raise_for_status()
|
||||
@@ -98,6 +104,8 @@ def call_llm_streaming(
|
||||
"stream": True,
|
||||
"options": {"num_ctx": 4096},
|
||||
"think": thinking,
|
||||
# Keep the chat model resident between calls (see call_llm_direct).
|
||||
"keep_alive": "30m",
|
||||
}
|
||||
|
||||
# Use ``with`` so the streaming response (and the underlying TCP
|
||||
@@ -201,6 +209,8 @@ def chat_with_messages(
|
||||
"stream": False,
|
||||
"options": {"num_ctx": 8192},
|
||||
"think": thinking,
|
||||
# Keep the chat model resident between turns (see call_llm_direct).
|
||||
"keep_alive": "30m",
|
||||
}
|
||||
if extra_options and isinstance(extra_options, dict):
|
||||
# Merge shallowly into options
|
||||
|
||||
@@ -6,7 +6,11 @@ def get_embedding(text: str, base_url: str, model: str, timeout_sec: float = 15.
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{base_url.rstrip('/')}/api/embeddings",
|
||||
json={"model": model, "prompt": text},
|
||||
# keep_alive=0 unloads the embedding model right after the call so
|
||||
# it does not sit resident in VRAM alongside the chat model. The
|
||||
# chat model is pinned separately (llm.py keep_alive=30m); only the
|
||||
# actively-used chat model should stay loaded.
|
||||
json={"model": model, "prompt": text, "keep_alive": 0},
|
||||
timeout=timeout_sec,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
Reference in New Issue
Block a user