diff --git a/apps/server/src/ai/brain.ts b/apps/server/src/ai/brain.ts index c20622e..2552c8b 100644 --- a/apps/server/src/ai/brain.ts +++ b/apps/server/src/ai/brain.ts @@ -207,6 +207,8 @@ export class BotBrain { onGameStart(): void { this.happened = []; + // A game with computers and people: make sure the model is loaded before anyone talks. + if (this.llm?.available() && this.room.humans().length && this.room.bots().length) void this.llm.warm(); if (!this.enabled() || this.room.gameId() === 'mafia' || this.random() > 0.7) return; const bots = this.room.bots(); const bot = bots[Math.floor(this.random() * bots.length)]; diff --git a/apps/server/src/ai/llm.ts b/apps/server/src/ai/llm.ts index e3b0cff..18ee8e5 100644 --- a/apps/server/src/ai/llm.ts +++ b/apps/server/src/ai/llm.ts @@ -83,7 +83,7 @@ export class Llm { model: this.cfg.model, messages, stream: false, - keep_alive: this.cfg.keepAlive ?? '30m', + keep_alive: this.cfg.keepAlive ?? '2h', ...(opts.format ? { format: opts.format } : {}), options: { temperature: opts.temperature ?? 0.9, num_predict: opts.maxTokens ?? 80 }, }), @@ -115,8 +115,15 @@ export class Llm { } } - /** Loads the model into GPU memory ahead of the first real request (the first load takes ~20 s). */ + private lastWarm = -Infinity; + + /** + * Loads the model into GPU memory ahead of the first real request (a cold load takes ~20 s; Ollama unloads it after + * `keepAlive` without use). Cheap when it is already loaded; at most once a minute. + */ async warm(): Promise { + if (this.now() - this.lastWarm < 60_000) return true; + this.lastWarm = this.now(); const r = await this.chat([{ role: 'user', content: '안녕' }], { maxTokens: 1, timeoutMs: 60_000 }); return r !== null; }