From b52ffd2b185c58edd40dc7562f8b2ed9e04d81eb Mon Sep 17 00:00:00 2001 From: javis-bot Date: Tue, 23 Jun 2026 17:35:40 +0900 Subject: [PATCH] perf: run auxiliary LLM calls on a small model, big model only for the answer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Intent judging, tool routing and arg extraction are classification/JSON calls, not the spoken answer, yet the stack aliased OLLAMA_INTENT_MODEL back to the big chat model — so each command paid the big model's cost 2-3 times for routing before the reply even ran. With the GPU on, that round-trip stacking is the main remaining per-turn latency. Default OLLAMA_INTENT_MODEL to qwen2.5:3b (the project's reference small model, clean Korean on classification) and have ollama-init pull it. The reply engine already routes these calls through intent_judge_model, so answer quality is untouched; set OLLAMA_INTENT_MODEL = OLLAMA_CHAT_MODEL to fold back onto one resident model. Co-Authored-By: Claude Opus 4.7 --- .env.example | 11 +++--- docker-compose.yml | 15 ++++++++ docker/entrypoint.sh | 12 +++---- tests/test_intent_model_split.py | 62 ++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 11 deletions(-) create mode 100644 tests/test_intent_model_split.py diff --git a/.env.example b/.env.example index 86eaaf4..332dbb2 100644 --- a/.env.example +++ b/.env.example @@ -59,11 +59,12 @@ OLLAMA_BASE_URL=http://127.0.0.1:11434 # free-form chit-chat. Swap back to qwen3:8b for the strongest tool-calling. OLLAMA_CHAT_MODEL=qwen2.5:3b # Model for the auxiliary small-model calls: intent judge, tool router, weather -# place extraction, query decomposition. BLANK (default) reuses OLLAMA_CHAT_MODEL -# so the stack runs on one already-warm model. The code's built-in default -# (gemma4:e2b) is NOT pulled by this stack, so leaving this unset previously made -# every router/extractor call silently fail. Only set this if you also pull the -# model into Ollama. +# place extraction, query decomposition. These are classification/JSON calls, +# NOT the spoken answer, so a small fast model here cuts 2-3 big-model round +# trips per command without touching answer quality. BLANK uses the stack +# default qwen2.5:3b, which ollama-init pulls automatically. Set it equal to +# OLLAMA_CHAT_MODEL to run everything on one resident model instead (saves VRAM +# at the cost of slower routing when the chat model is large). OLLAMA_INTENT_MODEL= OLLAMA_EMBED_MODEL=nomic-embed-text WHISPER_MODEL=small diff --git a/docker-compose.yml b/docker-compose.yml index fb7a549..395683f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,9 @@ services: environment: OLLAMA_HOST: http://ollama:11434 CHAT_MODEL: ${OLLAMA_CHAT_MODEL:-qwen2.5:3b} + # Small auxiliary model for intent/router/extraction calls (see javis + # service). Pulled here so the split is ready out of the box. + INTENT_MODEL: ${OLLAMA_INTENT_MODEL:-qwen2.5:3b} EMBED_MODEL: ${OLLAMA_EMBED_MODEL:-nomic-embed-text} entrypoint: ["/bin/sh", "-c"] command: @@ -48,6 +51,10 @@ services: until ollama list >/dev/null 2>&1; do sleep 2; done; echo "[ollama-init] pulling $$CHAT_MODEL"; ollama pull "$$CHAT_MODEL"; + if [ -n "$$INTENT_MODEL" ] && [ "$$INTENT_MODEL" != "$$CHAT_MODEL" ]; then + echo "[ollama-init] pulling $$INTENT_MODEL (auxiliary intent/router model)"; + ollama pull "$$INTENT_MODEL"; + fi; echo "[ollama-init] pulling $$EMBED_MODEL"; ollama pull "$$EMBED_MODEL"; echo "[ollama-init] models ready."; @@ -62,6 +69,14 @@ services: # Point the brain at the ollama service and the bot at the in-container bridge. OLLAMA_BASE_URL: http://ollama:11434 OLLAMA_CHAT_MODEL: ${OLLAMA_CHAT_MODEL:-qwen2.5:3b} + # Auxiliary small-model calls (intent judge, tool router, arg extraction, + # query decomposition) run on this fast model so the big chat model only + # runs for the actual spoken answer. With the GPU on, this is the main + # per-turn latency win: a command no longer pays the big model's cost 2-3 + # times for routing/extraction. Defaults to qwen2.5:3b (the project's + # reference small model, clean Korean on classification); set it equal to + # OLLAMA_CHAT_MODEL to fold everything back onto one resident model. + OLLAMA_INTENT_MODEL: ${OLLAMA_INTENT_MODEL:-qwen2.5:3b} OLLAMA_EMBED_MODEL: ${OLLAMA_EMBED_MODEL:-nomic-embed-text} WHISPER_MODEL: ${WHISPER_MODEL:-medium} WHISPER_DEVICE: ${WHISPER_DEVICE:-cuda} diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 23bbabc..a1fa5e3 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -10,12 +10,12 @@ set -euo pipefail : "${OLLAMA_BASE_URL:=http://ollama:11434}" : "${OLLAMA_CHAT_MODEL:=qwen3:8b}" # Auxiliary small-model calls (intent judge, tool router, weather place -# extraction, query decomposition). The code default is gemma4:e2b, which this -# stack does not pull, so those calls would silently fail and fall open — -# crippling tool routing and arg extraction. Reuse the (already warm) chat model -# by default so everything runs on one resident model; override if you pull a -# dedicated small model. -: "${OLLAMA_INTENT_MODEL:=${OLLAMA_CHAT_MODEL}}" +# extraction, query decomposition). Default to a small fast model so the big +# chat model only runs for the actual spoken answer — the main per-turn latency +# win once the GPU is in use, since the 2-3 routing/extraction calls per command +# no longer pay the big model's cost. ollama-init pulls this model. Set it equal +# to OLLAMA_CHAT_MODEL to fold everything back onto one resident model. +: "${OLLAMA_INTENT_MODEL:=qwen2.5:3b}" # Cap chat-model output tokens per turn (worst-case latency guard). Spoken # answers are 1-2 sentences; 512 is safe headroom above tool-call JSON. 0 = off. : "${OLLAMA_NUM_PREDICT:=512}" diff --git a/tests/test_intent_model_split.py b/tests/test_intent_model_split.py new file mode 100644 index 0000000..30a38ef --- /dev/null +++ b/tests/test_intent_model_split.py @@ -0,0 +1,62 @@ +"""The docker deployment must run auxiliary calls on a small model. + +Latency win: intent judging, tool routing and arg extraction are +classification/JSON calls, not the spoken answer. Running them on a small fast +model means the big chat model only runs once per command (for the answer), +instead of 2-3 times per command for routing/extraction too. + +The wiring is: docker/jarvis-config.template.json renders `intent_judge_model` +from `${OLLAMA_INTENT_MODEL}`, and the reply engine's resolver falls through +`tool_router_model -> intent_judge_model -> ollama_chat_model`. The template +sets no `tool_router_model`, so the auxiliary model is whatever +`OLLAMA_INTENT_MODEL` renders to. These tests pin that behaviour end to end. +""" + +import json +import string +from pathlib import Path + +import pytest + +from jarvis.reply.engine import resolve_tool_router_model + +TEMPLATE = Path(__file__).resolve().parent.parent / "docker" / "jarvis-config.template.json" + + +def _render(**env) -> dict: + raw = TEMPLATE.read_text(encoding="utf-8") + return json.loads(string.Template(raw).safe_substitute(**env)) + + +class _Cfg: + """cfg stand-in carrying only the fields the resolver reads. The template + does not render `tool_router_model`, so it stays empty here too.""" + + def __init__(self, rendered: dict): + self.tool_router_model = rendered.get("tool_router_model", "") or "" + self.intent_judge_model = rendered.get("intent_judge_model", "") or "" + self.ollama_chat_model = rendered.get("ollama_chat_model", "") or "" + + +def test_template_renders_separate_intent_model(): + cfg = _render(OLLAMA_CHAT_MODEL="qwen3:8b", OLLAMA_INTENT_MODEL="qwen2.5:3b") + assert cfg["ollama_chat_model"] == "qwen3:8b" + assert cfg["intent_judge_model"] == "qwen2.5:3b" + assert cfg["intent_judge_model"] != cfg["ollama_chat_model"] + + +@pytest.mark.unit +def test_aux_calls_route_to_small_model_not_chat_model(): + """The whole point: with a big chat model and a small intent model, tool + routing must resolve to the small model, leaving the big model for answers.""" + cfg = _Cfg(_render(OLLAMA_CHAT_MODEL="qwen3:8b", OLLAMA_INTENT_MODEL="qwen2.5:3b")) + assert resolve_tool_router_model(cfg) == "qwen2.5:3b" + + +@pytest.mark.unit +def test_folding_intent_onto_chat_model_keeps_one_model(): + """Setting OLLAMA_INTENT_MODEL == OLLAMA_CHAT_MODEL folds everything back + onto a single resident model (the documented VRAM-saving opt-out).""" + cfg = _Cfg(_render(OLLAMA_CHAT_MODEL="qwen2.5:3b", OLLAMA_INTENT_MODEL="qwen2.5:3b")) + assert resolve_tool_router_model(cfg) == "qwen2.5:3b" + assert cfg.intent_judge_model == cfg.ollama_chat_model