From 5c11c5f7e82ec0b77bdddd5dd148c1a5cd1e2707 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Sun, 14 Jun 2026 00:25:34 +0900 Subject: [PATCH] perf: pre-warm ollama at the engine's num_ctx (8192) so first turn is hot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The startup warm-up loaded qwen at the default context, but the reply engine chats at num_ctx=8192 — a different Ollama instance — so the first real turn still cold-reloaded. Warm at OLLAMA_NUM_CTX via /api/chat. Co-Authored-By: Claude Opus 4.7 --- bridge/server.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/bridge/server.py b/bridge/server.py index e05095d..015cb47 100644 --- a/bridge/server.py +++ b/bridge/server.py @@ -527,23 +527,31 @@ def http_converse_stream(): def _warm_ollama(base_url: str, model: str) -> None: """Load ``model`` into Ollama (GPU if available) with a long keep_alive so it - is resident before the first real turn. Best-effort.""" + is resident before the first real turn. Best-effort. + + Warms at the SAME num_ctx the reply engine uses (OLLAMA_NUM_CTX, default + 8192). Ollama keeps a distinct loaded instance per (model, num_ctx), so + warming at the default context would load the wrong instance and the first + real chat call (8192) would still cold-reload (~3.4s).""" if not base_url or not model: return import urllib.request + num_ctx = int(os.environ.get("OLLAMA_NUM_CTX", "8192")) try: req = urllib.request.Request( - f"{base_url.rstrip('/')}/api/generate", + f"{base_url.rstrip('/')}/api/chat", data=json.dumps( - {"model": model, "prompt": "", "stream": False, - "keep_alive": "30m", "options": {"num_predict": 1}} + {"model": model, + "messages": [{"role": "user", "content": "."}], + "stream": False, "keep_alive": "30m", + "options": {"num_ctx": num_ctx, "num_predict": 1}} ).encode("utf-8"), headers={"Content-Type": "application/json"}, ) with urllib.request.urlopen(req, timeout=120) as resp: ok = resp.status == 200 - print(f"[bridge] {'✅' if ok else '⚠️'} ollama warm (model={model})", flush=True) + print(f"[bridge] {'✅' if ok else '⚠️'} ollama warm (model={model}, num_ctx={num_ctx})", flush=True) except Exception as e: # pragma: no cover - depends on local ollama print(f"[bridge] ollama warmup skipped (model={model}): {e}", flush=True)