tkrmagid/javis_bot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 20 Wiki Activity

perf: pre-warm ollama at the engine's num_ctx (8192) so first turn is hot

Browse Source 
The startup warm-up loaded qwen at the default context, but the reply
engine chats at num_ctx=8192 — a different Ollama instance — so the first
real turn still cold-reloaded. Warm at OLLAMA_NUM_CTX via /api/chat.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-06-14 00:25:34 +09:00
parent 2c38e7576d
commit 5c11c5f7e8
1 changed files with 13 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
bridge/server.py
View File

@@ -527,23 +527,31 @@ def http_converse_stream():
def _warm_ollama(base_url: str, model: str) -> None: def _warm_ollama(base_url: str, model: str) -> None:
"""Load ``model`` into Ollama (GPU if available) with a long keep_alive so it """Load ``model`` into Ollama (GPU if available) with a long keep_alive so it
is resident before the first real turn. Best-effort.""" is resident before the first real turn. Best-effort.
Warms at the SAME num_ctx the reply engine uses (OLLAMA_NUM_CTX, default
8192). Ollama keeps a distinct loaded instance per (model, num_ctx), so
warming at the default context would load the wrong instance and the first
real chat call (8192) would still cold-reload (~3.4s)."""
if not base_url or not model: if not base_url or not model:
return return
import urllib.request import urllib.request
num_ctx = int(os.environ.get("OLLAMA_NUM_CTX", "8192"))
try: try:
req = urllib.request.Request( req = urllib.request.Request(
f"{base_url.rstrip('/')}/api/generate", f"{base_url.rstrip('/')}/api/chat",
data=json.dumps( data=json.dumps(
{"model": model, "prompt": "", "stream": False, {"model": model,
"keep_alive": "30m", "options": {"num_predict": 1}} "messages": [{"role": "user", "content": "."}],
"stream": False, "keep_alive": "30m",
"options": {"num_ctx": num_ctx, "num_predict": 1}}
).encode("utf-8"), ).encode("utf-8"),
headers={"Content-Type": "application/json"}, headers={"Content-Type": "application/json"},
) )
with urllib.request.urlopen(req, timeout=120) as resp: with urllib.request.urlopen(req, timeout=120) as resp:
ok = resp.status == 200 ok = resp.status == 200
print(f"[bridge] {'✅' if ok else '⚠️'} ollama warm (model={model})", flush=True) print(f"[bridge] {'✅' if ok else '⚠️'} ollama warm (model={model}, num_ctx={num_ctx})", flush=True)
except Exception as e: # pragma: no cover - depends on local ollama except Exception as e: # pragma: no cover - depends on local ollama
print(f"[bridge] ollama warmup skipped (model={model}): {e}", flush=True) print(f"[bridge] ollama warmup skipped (model={model}): {e}", flush=True)