Files
javis_bot/scripts/m0/m0lib/llm_bench.py
javis-bot 653d6ecc03 feat(m0): add gaming-PC measurement kit for v2 defaults
One double-click kit (scripts/m0/run_m0.bat) that measures the gaming PC
and packs the results into a single zip:

- PC info (GPU, driver, RAM, disks, monitors)
- Windows checks with the user: bot browser window hidden from Alt+Tab
  but listed in Discord's share picker and still drawing while covered;
  Discord pop-out stream window identification and occlusion; installed
  games, anti-cheat services and the borderless game detector
- MeloTTS, CosyVoice2 and Fun-CosyVoice3 Korean synthesis speed
- faster-whisper large-v3-turbo: final/partial latency, CER, wake-word
  hint, batched decoding for several speakers
- Strata LLM (fast and recommended presets): TTFT by context length,
  decode speed, tool calls, picture reading per quality tier, Korean
  answers, parallel requests, VRAM give-back and restart/switch timings

uv and Python, per-stage venvs, models and Strata all live inside the
work folder. Results are scrubbed of user/computer names and home paths.
Verified on Linux GPU (LLM client against Ollama, STT, MeloTTS,
CosyVoice2/3) and on the Windows 11 CI VM (window checks, game scan,
MeloTTS/CosyVoice2/STT installs, Strata install and plan).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-09 15:38:41 +09:00

383 lines
19 KiB
Python

"""LLM measurements against an OpenAI-compatible server (Strata or another).
Every test is fail-open: an error is recorded in the result and the next test
runs. The answers are kept so Korean quality can be read afterwards.
"""
from __future__ import annotations
import base64
import io
import json
import os
import random
import threading
import time
from pathlib import Path
from typing import Any
from m0lib.common import StageResult, gpu_memory_mib, say, stats
from m0lib.llm_client import ChatClient, http_json
NO_THINK = {"chat_template_kwargs": {"enable_thinking": False}}
VOICE_SYSTEM = ("너는 디스코드 음성 채널에서 대답하는 비서 '자비스'다. 대답은 소리 내어 읽히므로 "
"한국어 존댓말로 짧게, 두 문장 이내로 말한다. 목록, 마크다운, 이모지는 쓰지 않는다.")
# --------------------------------------------------------------------------
# Context length / TTFT
_WORDS = ("오늘 내일 회의 일정 날씨 음악 게임 친구 저녁 메뉴 영화 주말 기록 화면 모니터 채널 "
"서버 사진 여행 공부 운동 커피 버스 지하철 약속 생일 선물 노래 방송 뉴스 시장 가격 "
"연습 준비 정리 확인 계획 결과 문제 해결 방법 이유 생각 이야기 질문 대답 시간 장소").split()
def filler(n_chars: int, seed: int) -> str:
rnd = random.Random(seed)
parts, size = [], 0
while size < n_chars:
s = " ".join(rnd.choice(_WORDS) for _ in range(rnd.randint(6, 12))) + "에 대해 이야기했다. "
parts.append(s)
size += len(s)
return "".join(parts)
def test_context(r: StageResult, c: ChatClient, ctx_limit: int) -> None:
say(" - 맥락 길이별 첫 토큰 시간")
rows = []
# Characters per token differ between tokenizers: start with a guess and
# correct it from the server's prompt_tokens after the first request.
chars_per_token = 1.5
for target in (256, 2048, 8192, 16384, 24576):
if target > ctx_limit * 0.8:
continue
seed = random.randint(0, 10**9)
doc = f"[문서 {seed}]\n" + filler(int(target * chars_per_token), seed)
msgs = [{"role": "system", "content": VOICE_SYSTEM},
{"role": "user", "content": doc + "\n\n위 글의 주제를 한 문장으로 말해 줘."}]
row: dict[str, Any] = {"target_tokens": target}
try:
a = c.chat(msgs, max_tokens=48, extra=NO_THINK)
row.update(prompt_tokens=(a["usage"] or {}).get("prompt_tokens"), ttft=a["ttft"],
first_sentence=a["first_sentence"], total=a["total"])
if row["prompt_tokens"]:
chars_per_token = len(doc) / row["prompt_tokens"]
# Same long prefix, a new question at the end: shows prompt-cache reuse.
msgs2 = msgs[:1] + [{"role": "user", "content": doc + "\n\n위 글에서 가장 많이 나온 낱말 하나만 말해 줘."}]
b = c.chat(msgs2, max_tokens=24, extra=NO_THINK)
row.update(ttft_same_prefix=b["ttft"])
say(f" {row.get('prompt_tokens')} 토큰: 첫 토큰 {row['ttft']}s, 같은 앞부분 재사용 {row['ttft_same_prefix']}s")
except Exception as e:
r.error(f"context {target}", e)
rows.append(row)
r["context_ttft"] = rows
def test_decode(r: StageResult, c: ChatClient, key: str = "decode") -> None:
say(" - 생성 속도 (tok/s)")
runs = []
for i in range(3):
try:
a = c.chat([{"role": "user", "content": f"({i}) 한국의 사계절을 각각 세 문장씩 자세히 설명해 줘."}],
max_tokens=320, extra=NO_THINK)
runs.append({k: a.get(k) for k in ("ttft", "total", "decode_tok_s", "usage", "server_timings")})
except Exception as e:
r.error(f"{key} {i}", e)
r[key] = {"runs": runs, "decode_tok_s": stats(x.get("decode_tok_s") for x in runs),
"ttft": stats(x.get("ttft") for x in runs)}
say(f" 중앙값 {r[key]['decode_tok_s'].get('median')} tok/s")
try:
code, body = http_json("GET", c.base + "/status", key=c.key, timeout=10)
if code == 200 and isinstance(body, dict):
r[f"{key}_server_last_timings"] = body.get("last_timings")
except Exception:
pass
def test_thinking_default(r: StageResult, c: ChatClient) -> None:
"""Does the model think by default, and what does that cost the first answer token?"""
try:
a = c.chat([{"role": "user", "content": "자비스, 오늘 저녁 메뉴 하나만 추천해 줘."}], max_tokens=400)
r["thinking_default"] = {k: a.get(k) for k in ("ttft", "first_reasoning", "reasoning_chars", "total", "text")}
except Exception as e:
r.error("thinking_default", e)
# --------------------------------------------------------------------------
# Tool calling
TOOLS = [
{"type": "function", "function": {
"name": "get_weather", "description": "지정한 도시의 현재 날씨를 가져온다.",
"parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "도시 이름"}},
"required": ["city"]}}},
{"type": "function", "function": {
"name": "set_timer", "description": "몇 분 뒤에 알리는 타이머를 맞춘다.",
"parameters": {"type": "object", "properties": {"minutes": {"type": "integer"}}, "required": ["minutes"]}}},
{"type": "function", "function": {
"name": "view_screen", "description": "사용자 PC의 화면을 캡처해서 본다.",
"parameters": {"type": "object", "properties": {
"monitor": {"type": "integer", "description": "모니터 번호 (1부터)"},
"quality": {"type": "string", "enum": ["fast", "normal", "detail"]}}, "required": ["monitor"]}}},
]
TOOL_CASES = [
("서울 날씨 어때?", "get_weather", lambda a: "서울" in str(a.get("city", "")) or "Seoul" in str(a.get("city", ""))),
("5분 타이머 맞춰 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 5),
("두 번째 모니터에 뭐 떠 있어?", "view_screen", lambda a: int(a.get("monitor", 0)) == 2),
("안녕, 오늘 기분 어때?", None, None),
]
def test_tools(r: StageResult, c: ChatClient) -> None:
say(" - 도구 호출")
rows = []
for prompt, want, check in TOOL_CASES:
for i in range(3):
row: dict[str, Any] = {"prompt": prompt, "expected": want, "try": i + 1}
try:
a = c.chat([{"role": "system", "content": VOICE_SYSTEM}, {"role": "user", "content": prompt}],
max_tokens=200, tools=TOOLS, extra=NO_THINK)
calls = a["tool_calls"]
row.update(total=a["total"], first_tool=a["first_tool"], ttft=a["ttft"], finish=a["finish"],
calls=calls, text=a["text"][:200])
if want is None:
row["ok"] = not calls
else:
ok = bool(calls) and calls[0]["name"] == want
if ok:
try:
ok = bool(check(json.loads(calls[0]["arguments"] or "{}")))
except Exception:
ok = False
row["ok"] = ok
except Exception as e:
row["error"] = str(e)[:300]
row["ok"] = False
rows.append(row)
ok = sum(1 for x in rows if x.get("ok"))
r["tools"] = {"cases": rows, "ok": ok, "total": len(rows)}
say(f" 성공 {ok}/{len(rows)}")
# Round trip: give the tool result back and read the final answer.
try:
msgs = [{"role": "system", "content": VOICE_SYSTEM}, {"role": "user", "content": "서울 날씨 어때?"}]
a = c.chat(msgs, max_tokens=200, tools=TOOLS, extra=NO_THINK)
if a["tool_calls"]:
call = a["tool_calls"][0]
msgs += [{"role": "assistant", "content": "", "tool_calls": [
{"id": "call_1", "type": "function",
"function": {"name": call["name"], "arguments": call["arguments"]}}]},
{"role": "tool", "tool_call_id": "call_1",
"content": json.dumps({"city": "서울", "temp_c": 13, "sky": "맑음"}, ensure_ascii=False)}]
b = c.chat(msgs, max_tokens=120, tools=TOOLS, extra=NO_THINK)
r["tools_round_trip"] = {"ttft": b["ttft"], "first_sentence": b["first_sentence"], "text": b["text"],
"uses_result": "13" in b["text"]}
except Exception as e:
r.error("tools round trip", e)
# --------------------------------------------------------------------------
# Pictures
CODE_BIG = "4827"
CODE_SMALL = "QX-7316"
def _font(size: int):
from PIL import ImageFont
for p in (r"C:\Windows\Fonts\malgun.ttf", r"C:\Windows\Fonts\malgunbd.ttf",
"/usr/share/fonts/truetype/nanum/NanumGothic.ttf",
"/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc"):
if os.path.exists(p):
return ImageFont.truetype(p, size)
return ImageFont.load_default(size=size)
def make_screen(seed: int = 0) -> "Any":
"""A synthetic 4K 'desktop' with a big number, a small code and Korean text.
``seed`` changes a few pixels' worth of content so a server cannot answer a
repeated try from its picture cache.
"""
from PIL import Image, ImageDraw
img = Image.new("RGB", (3840, 2160), (32, 56, 92))
d = ImageDraw.Draw(img)
d.rectangle([200, 160, 2400, 1500], fill=(245, 245, 245))
d.rectangle([200, 160, 2400, 230], fill=(220, 220, 220))
d.text((230, 172), "메모장 - 오늘 할 일.txt", font=_font(40), fill=(20, 20, 20))
lines = ["1. 오후 세 시 회의 자료 정리", "2. 저녁 장보기: 두부, 계란, 대파", "3. 디스코드 서버 공지 올리기"]
for i, line in enumerate(lines):
d.text((260, 300 + i * 90), line, font=_font(56), fill=(30, 30, 30))
d.rectangle([2700, 300, 3500, 800], outline=(220, 30, 30), width=14)
d.text((2830, 420), CODE_BIG, font=_font(220), fill=(220, 30, 30))
d.text((260, 1400), f"확인 코드: {CODE_SMALL}", font=_font(26), fill=(90, 90, 90))
d.rectangle([0, 2080, 3840, 2160], fill=(20, 20, 20))
d.text((3300, 2095), f"오후 {seed % 12 + 1}:{seed % 60:02d}", font=_font(44), fill=(230, 230, 230))
return img
def _jpeg_b64(img: Any, long_side: int) -> tuple[str, float, tuple[int, int]]:
t0 = time.perf_counter()
w, h = img.size
s = long_side / max(w, h)
im = img.resize((max(1, round(w * s)), max(1, round(h * s)))) if s < 1 else img
buf = io.BytesIO()
im.save(buf, "JPEG", quality=85)
return base64.b64encode(buf.getvalue()).decode(), time.perf_counter() - t0, im.size
def picture_parts(img: Any, tier: str) -> tuple[list[dict], float, list]:
if tier == "detail":
w, h = img.size
tiles = [img.crop((x, y, x + w // 2, y + h // 2)) for y in (0, h // 2) for x in (0, w // 2)]
enc = [_jpeg_b64(t, 1280) for t in tiles]
else:
enc = [_jpeg_b64(img, {"fast": 768, "normal": 1280}[tier])]
parts = [{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b}"}} for b, _, _ in enc]
return parts, sum(t for _, t, _ in enc), [s for _, _, s in enc]
def test_images(r: StageResult, c: ChatClient, key: str = "images") -> None:
say(" - 이미지 읽기 (화질 단계별)")
rows = []
question = ("이 화면에서 빨간 상자 안의 숫자와, 작은 글씨로 적힌 '확인 코드'를 읽어 줘. "
"모르면 모른다고 해. 형식: 숫자=..., 코드=...")
seed = random.randint(0, 10**6)
for tier in ("fast", "normal", "detail"):
for i in range(3):
seed += 1
# The first picture after a start may include starting the picture reader.
row: dict[str, Any] = {"tier": tier, "try": i + 1}
try:
parts, prep, sizes = picture_parts(make_screen(seed), tier)
msgs = [{"role": "user", "content": parts + [{"type": "text", "text": question}]}]
a = c.chat(msgs, max_tokens=60, extra=NO_THINK)
row.update(prep_s=round(prep, 3), sizes=sizes, ttft=a["ttft"], total=a["total"],
prompt_tokens=(a["usage"] or {}).get("prompt_tokens"), text=a["text"][:200],
big_ok=CODE_BIG in a["text"], small_ok=CODE_SMALL.replace("-", "") in
a["text"].replace("-", "").replace(" ", "").upper())
say(f" {tier}: 첫 토큰 {row['ttft']}s, 큰 숫자 {'O' if row['big_ok'] else 'X'}, "
f"작은 코드 {'O' if row['small_ok'] else 'X'}")
except Exception as e:
row["error"] = str(e)[:300]
say(f" {tier}: 실패 ({row['error'][:120]})")
rows.append(row)
r[key] = rows
r[key + "_summary"] = {t: {"ttft_after_first": stats(x.get("ttft") for x in rows if x["tier"] == t and x["try"] > 1),
"big_ok": sum(1 for x in rows if x["tier"] == t and x.get("big_ok")),
"small_ok": sum(1 for x in rows if x["tier"] == t and x.get("small_ok")),
"tries": sum(1 for x in rows if x["tier"] == t)}
for t in ("fast", "normal", "detail")}
# --------------------------------------------------------------------------
# Korean quality: answers are saved for reading
KOREAN_PROMPTS = [
("voice", "자비스, 오늘 비 온다는데 우산 챙겨야 할까?"),
("voice", "자비스, 라면 맛있게 끓이는 방법 알려 줘."),
("voice", "자비스, 아까 내가 무슨 노래 틀어 달라고 했지? (기억이 없으면 솔직하게 말해 줘)"),
("voice", "자비스, 3 곱하기 17 더하기 9는 얼마야?"),
("plain", "존댓말과 반말의 차이를 초등학생에게 설명하듯 세 문장으로 말해 줘."),
("plain", "다음 문장을 자연스러운 한국어로 고쳐 줘: '나는 어제 친구를 만나는 것을 했다 그리고 영화를 보았다.'"),
("plain", "\"내일 오후 2시에 강남역에서 민수랑 만나기로 했어\"에서 시간, 장소, 사람을 JSON으로 뽑아 줘."),
("plain", "Translate to natural Korean: 'I'll be right back, just grabbing a coffee.'"),
]
def test_korean(r: StageResult, c: ChatClient) -> None:
say(" - 한국어 답변 모음")
rows = []
for kind, prompt in KOREAN_PROMPTS:
msgs = ([{"role": "system", "content": VOICE_SYSTEM}] if kind == "voice" else []) + \
[{"role": "user", "content": prompt}]
try:
a = c.chat(msgs, max_tokens=300, extra=NO_THINK)
rows.append({"kind": kind, "prompt": prompt, "answer": a["text"], "ttft": a["ttft"],
"first_sentence": a["first_sentence"], "total": a["total"]})
except Exception as e:
rows.append({"kind": kind, "prompt": prompt, "error": str(e)[:300]})
r["korean"] = rows
r["voice_first_sentence"] = stats(x.get("first_sentence") for x in rows if x["kind"] == "voice")
# --------------------------------------------------------------------------
# Several requests at once
def test_parallel(r: StageResult, c: ChatClient, n_list=(2, 4)) -> None:
say(" - 동시 요청")
out = {}
for n in n_list:
results: list[dict] = [{} for _ in range(n)]
def one(i: int) -> None:
try:
a = c.chat([{"role": "user", "content": f"({i}) 좋아하는 계절 하나를 골라 이유를 다섯 문장으로 말해 줘."}],
max_tokens=160, extra=NO_THINK)
results[i] = {k: a.get(k) for k in ("ttft", "total", "decode_tok_s")}
results[i]["completion_tokens"] = (a.get("usage") or {}).get("completion_tokens")
except Exception as e:
results[i] = {"error": str(e)[:200]}
t0 = time.perf_counter()
th = [threading.Thread(target=one, args=(i,)) for i in range(n)]
for t in th:
t.start()
for t in th:
t.join()
wall = time.perf_counter() - t0
toks = sum(x.get("completion_tokens") or 0 for x in results)
out[str(n)] = {"wall_s": round(wall, 2), "sum_tokens": toks, "aggregate_tok_s": round(toks / wall, 1),
"ttft": stats(x.get("ttft") for x in results), "requests": results}
say(f" {n}개 동시: 전체 {out[str(n)]['aggregate_tok_s']} tok/s, "
f"첫 토큰 최대 {out[str(n)]['ttft'].get('max')}s")
r["parallel"] = out
# --------------------------------------------------------------------------
# Game mode (a): give VRAM back with POST /v1/vram and take it again
def test_vram_shrink(r: StageResult, c: ChatClient, reserve_mib: int = 8000) -> None:
say(f" - 게임 모드 (a): VRAM {reserve_mib} MiB 비우기")
out: dict[str, Any] = {"reserve_mib": reserve_mib, "gpu_before": gpu_memory_mib()}
root = c.base[:-3] if c.base.endswith("/v1") else c.base
try:
t0 = time.perf_counter()
code, body = http_json("POST", root + "/v1/vram", {"reserve_mib": reserve_mib}, timeout=400, key=c.key)
out["shrink"] = {"http": code, "seconds": round(time.perf_counter() - t0, 3), "body": body}
out["gpu_after_shrink"] = gpu_memory_mib()
if code == 200:
a = c.chat([{"role": "user", "content": "한국의 사계절을 각각 세 문장씩 설명해 줘."}],
max_tokens=200, extra=NO_THINK)
out["decode_tok_s_shrunk"] = a.get("decode_tok_s")
out["ttft_shrunk"] = a.get("ttft")
t0 = time.perf_counter()
code2, body2 = http_json("POST", root + "/v1/vram", {"reserve_mib": None}, timeout=400, key=c.key)
out["grow"] = {"http": code2, "seconds": round(time.perf_counter() - t0, 3), "body": body2}
out["gpu_after_grow"] = gpu_memory_mib()
say(f" 비우기 HTTP {code} ({out['shrink']['seconds']}s), 되돌리기 HTTP {code2} ({out['grow']['seconds']}s), "
f"비운 상태 속도 {out.get('decode_tok_s_shrunk')} tok/s")
except Exception as e:
r.error("vram shrink", e)
r["vram_shrink"] = out
def warmup(r: StageResult, c: ChatClient) -> None:
"""One throwaway request so model loading is not counted as answer time."""
try:
a = c.chat([{"role": "user", "content": "안녕"}], max_tokens=8, extra=NO_THINK)
r["warmup_total_s"] = a["total"]
except Exception as e:
r.error("warmup", e)
def bench_single(r: StageResult, c: ChatClient, ctx_limit: int) -> None:
"""The tests that need one request at a time."""
for fn in (lambda: warmup(r, c), lambda: test_decode(r, c), lambda: test_thinking_default(r, c), lambda: test_context(r, c, ctx_limit),
lambda: test_tools(r, c), lambda: test_images(r, c), lambda: test_korean(r, c)):
try:
fn()
except Exception as e:
r.error("bench", e)
r.save()