perf: pre-warm ollama at the engine's num_ctx (8192) so first turn is hot
The startup warm-up loaded qwen at the default context, but the reply engine chats at num_ctx=8192 — a different Ollama instance — so the first real turn still cold-reloaded. Warm at OLLAMA_NUM_CTX via /api/chat. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -527,23 +527,31 @@ def http_converse_stream():
|
||||
|
||||
def _warm_ollama(base_url: str, model: str) -> None:
|
||||
"""Load ``model`` into Ollama (GPU if available) with a long keep_alive so it
|
||||
is resident before the first real turn. Best-effort."""
|
||||
is resident before the first real turn. Best-effort.
|
||||
|
||||
Warms at the SAME num_ctx the reply engine uses (OLLAMA_NUM_CTX, default
|
||||
8192). Ollama keeps a distinct loaded instance per (model, num_ctx), so
|
||||
warming at the default context would load the wrong instance and the first
|
||||
real chat call (8192) would still cold-reload (~3.4s)."""
|
||||
if not base_url or not model:
|
||||
return
|
||||
import urllib.request
|
||||
|
||||
num_ctx = int(os.environ.get("OLLAMA_NUM_CTX", "8192"))
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
f"{base_url.rstrip('/')}/api/generate",
|
||||
f"{base_url.rstrip('/')}/api/chat",
|
||||
data=json.dumps(
|
||||
{"model": model, "prompt": "", "stream": False,
|
||||
"keep_alive": "30m", "options": {"num_predict": 1}}
|
||||
{"model": model,
|
||||
"messages": [{"role": "user", "content": "."}],
|
||||
"stream": False, "keep_alive": "30m",
|
||||
"options": {"num_ctx": num_ctx, "num_predict": 1}}
|
||||
).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=120) as resp:
|
||||
ok = resp.status == 200
|
||||
print(f"[bridge] {'✅' if ok else '⚠️'} ollama warm (model={model})", flush=True)
|
||||
print(f"[bridge] {'✅' if ok else '⚠️'} ollama warm (model={model}, num_ctx={num_ctx})", flush=True)
|
||||
except Exception as e: # pragma: no cover - depends on local ollama
|
||||
print(f"[bridge] ollama warmup skipped (model={model}): {e}", flush=True)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user