- 대답: 이름·이어 말하기·대화 중인 컴퓨터에 매번 대답, 입력 중에 온 말은 끝나고 대답, 모델 2.5초만 기다리고 내장 대답, 한 사람에게 1분 8번까지 - 모델 대기열 1칸(GPU 하나), 대답이 혼잣말보다 먼저, 혼잣말은 바쁘면 버림, 타자 0.7~3.5초 - 혼잣말: 센 베팅·다이·고/스톱 등 눈에 띈 수에만, 방 40초·컴퓨터 90초 간격, 시작 인사·끼어들기 없앰, 수의 뜻과 판 공개 정보(판돈·칩·점수)를 모델에 줌 - 수 종류(센 베팅·콜·체크/삥·다이/폴드·과감·안전·기권)별 내장 한마디, 어긋나거나 게임과 상관없는 말은 그걸로 바꿈 - 막말·지시문 되풀이 거름, 마피아·라이어 게임은 계획된 대사만 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
149 lines
5.7 KiB
TypeScript
149 lines
5.7 KiB
TypeScript
/**
|
|
* Local language model on the GPU (Ollama, docs/11 §8). Used by computer players to talk and think out loud
|
|
* (docs/06 §12.1). Everything here is best effort: when the model is slow, busy or down the caller gets `null`
|
|
* and the computers fall back to their built-in lines — a game never waits on the model.
|
|
*/
|
|
|
|
export interface LlmMessage {
|
|
role: 'system' | 'user' | 'assistant';
|
|
content: string;
|
|
}
|
|
|
|
export interface LlmConfig {
|
|
/** Ollama base URL, e.g. http://192.168.10.9:11434. */
|
|
url: string;
|
|
model: string;
|
|
/**
|
|
* Requests sent at once. Default 1: Ollama on the one GPU runs one request at a time (`n_slots = 1`), so a second
|
|
* one would only wait inside Ollama — where an answer to a person could not get ahead of small talk.
|
|
*/
|
|
maxConcurrent?: number;
|
|
/** Requests allowed to wait; more are refused at once (the caller uses a built-in line). */
|
|
maxQueue?: number;
|
|
timeoutMs?: number;
|
|
/** How long Ollama keeps the model in GPU memory after the last request. */
|
|
keepAlive?: string;
|
|
fetch?: typeof fetch;
|
|
log?: (level: 'info' | 'warn' | 'error', msg: string, extra?: Record<string, unknown>) => void;
|
|
now?: () => number;
|
|
}
|
|
|
|
export interface LlmStats {
|
|
model: string;
|
|
requests: number;
|
|
ok: number;
|
|
failed: number;
|
|
/** Refused because the queue was full or the model was backing off. */
|
|
skipped: number;
|
|
avgMs: number;
|
|
lastError: string | null;
|
|
backingOffUntil: number | null;
|
|
}
|
|
|
|
export class Llm {
|
|
private running = 0;
|
|
private waiting: { go: () => void; high: boolean }[] = [];
|
|
private totalMs = 0;
|
|
private backoffUntil = 0;
|
|
private failures = 0;
|
|
readonly stats: LlmStats;
|
|
|
|
constructor(private cfg: LlmConfig) {
|
|
this.stats = { model: cfg.model, requests: 0, ok: 0, failed: 0, skipped: 0, avgMs: 0, lastError: null, backingOffUntil: null };
|
|
}
|
|
|
|
private now(): number {
|
|
return this.cfg.now ? this.cfg.now() : Date.now();
|
|
}
|
|
|
|
/** False while backing off after errors (then callers don't even try). */
|
|
available(): boolean {
|
|
return this.now() >= this.backoffUntil;
|
|
}
|
|
|
|
/**
|
|
* One chat completion. `format` is a JSON schema for structured answers. Returns the text, or null when it could
|
|
* not be had in time (busy, down, timeout). `priority`: `high` (answering a person) goes ahead of everything that
|
|
* waits; `low` (small talk nobody asked for) is dropped at once when the model is busy, so it never delays the rest.
|
|
* `stop` ends the answer early (one chat line: a newline).
|
|
*/
|
|
async chat(
|
|
messages: LlmMessage[],
|
|
opts: { maxTokens?: number; temperature?: number; format?: object; timeoutMs?: number; priority?: 'high' | 'normal' | 'low'; stop?: string[] } = {},
|
|
): Promise<string | null> {
|
|
this.stats.requests++;
|
|
const max = this.cfg.maxConcurrent ?? 1;
|
|
const busy = this.running >= max;
|
|
const full = busy && this.waiting.length >= (this.cfg.maxQueue ?? 6) && opts.priority !== 'high';
|
|
if (!this.available() || full || (busy && opts.priority === 'low')) {
|
|
this.stats.skipped++;
|
|
return null;
|
|
}
|
|
if (busy) {
|
|
await new Promise<void>((go) => {
|
|
const high = opts.priority === 'high';
|
|
// Answers wait only behind earlier answers.
|
|
const at = high ? this.waiting.findIndex((w) => !w.high) : -1;
|
|
if (at >= 0) this.waiting.splice(at, 0, { go, high });
|
|
else this.waiting.push({ go, high });
|
|
});
|
|
} else this.running++; // a waiter is handed the finished request's place (`running` stays)
|
|
const t0 = this.now();
|
|
const ctl = new AbortController();
|
|
const timer = setTimeout(() => ctl.abort(), opts.timeoutMs ?? this.cfg.timeoutMs ?? 6000);
|
|
try {
|
|
const res = await (this.cfg.fetch ?? fetch)(`${this.cfg.url.replace(/\/$/, '')}/api/chat`, {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
signal: ctl.signal,
|
|
body: JSON.stringify({
|
|
model: this.cfg.model,
|
|
messages,
|
|
stream: false,
|
|
keep_alive: this.cfg.keepAlive ?? '2h',
|
|
...(opts.format ? { format: opts.format } : {}),
|
|
options: { temperature: opts.temperature ?? 0.9, num_predict: opts.maxTokens ?? 80, ...(opts.stop ? { stop: opts.stop } : {}) },
|
|
}),
|
|
});
|
|
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
const body = (await res.json()) as { message?: { content?: string } };
|
|
const text = body.message?.content ?? '';
|
|
const ms = this.now() - t0;
|
|
this.stats.ok++;
|
|
this.totalMs += ms;
|
|
this.stats.avgMs = Math.round(this.totalMs / this.stats.ok);
|
|
this.failures = 0;
|
|
return text;
|
|
} catch (err) {
|
|
this.stats.failed++;
|
|
this.stats.lastError = String(err).slice(0, 200);
|
|
this.failures++;
|
|
// A few failures in a row: stop asking for a while (30 s, up to 5 min) instead of slowing every move.
|
|
if (this.failures >= 3) {
|
|
this.backoffUntil = this.now() + Math.min(300_000, 30_000 * 2 ** (this.failures - 3));
|
|
this.stats.backingOffUntil = this.backoffUntil;
|
|
this.cfg.log?.('warn', 'ai backing off', { failures: this.failures, err: this.stats.lastError });
|
|
}
|
|
return null;
|
|
} finally {
|
|
clearTimeout(timer);
|
|
const next = this.waiting.shift();
|
|
if (next) next.go();
|
|
else this.running--;
|
|
}
|
|
}
|
|
|
|
private lastWarm = -Infinity;
|
|
|
|
/**
|
|
* Loads the model into GPU memory ahead of the first real request (a cold load takes ~20 s; Ollama unloads it after
|
|
* `keepAlive` without use). Cheap when it is already loaded; at most once a minute.
|
|
*/
|
|
async warm(): Promise<boolean> {
|
|
if (this.now() - this.lastWarm < 60_000) return true;
|
|
this.lastWarm = this.now();
|
|
const r = await this.chat([{ role: 'user', content: '안녕' }], { maxTokens: 1, timeoutMs: 60_000 });
|
|
return r !== null;
|
|
}
|
|
}
|