"""LLM measurements against an OpenAI-compatible server (Strata or another). Every test is fail-open: an error is recorded in the result and the next test runs. The answers are kept so Korean quality can be read afterwards. """ from __future__ import annotations import base64 import io import json import os import random import threading import time from pathlib import Path from typing import Any from m0lib.common import StageResult, gpu_memory_mib, say, stats from m0lib.llm_client import ChatClient, http_json NO_THINK = {"chat_template_kwargs": {"enable_thinking": False}} VOICE_SYSTEM = ("너는 디스코드 음성 채널에서 대답하는 비서 '자비스'다. 대답은 소리 내어 읽히므로 " "한국어 존댓말로 짧게, 두 문장 이내로 말한다. 목록, 마크다운, 이모지는 쓰지 않는다.") # -------------------------------------------------------------------------- # Context length / TTFT _WORDS = ("오늘 내일 회의 일정 날씨 음악 게임 친구 저녁 메뉴 영화 주말 기록 화면 모니터 채널 " "서버 사진 여행 공부 운동 커피 버스 지하철 약속 생일 선물 노래 방송 뉴스 시장 가격 " "연습 준비 정리 확인 계획 결과 문제 해결 방법 이유 생각 이야기 질문 대답 시간 장소").split() def filler(n_chars: int, seed: int) -> str: rnd = random.Random(seed) parts, size = [], 0 while size < n_chars: s = " ".join(rnd.choice(_WORDS) for _ in range(rnd.randint(6, 12))) + "에 대해 이야기했다. " parts.append(s) size += len(s) return "".join(parts) def test_context(r: StageResult, c: ChatClient, ctx_limit: int) -> None: say(" - 맥락 길이별 첫 토큰 시간") rows = [] # Characters per token differ between tokenizers: start with a guess and # correct it from the server's prompt_tokens after the first request. chars_per_token = 1.5 for target in (256, 2048, 8192, 16384, 24576): if target > ctx_limit * 0.8: continue seed = random.randint(0, 10**9) doc = f"[문서 {seed}]\n" + filler(int(target * chars_per_token), seed) msgs = [{"role": "system", "content": VOICE_SYSTEM}, {"role": "user", "content": doc + "\n\n위 글의 주제를 한 문장으로 말해 줘."}] row: dict[str, Any] = {"target_tokens": target} try: a = c.chat(msgs, max_tokens=48, extra=NO_THINK) row.update(prompt_tokens=(a["usage"] or {}).get("prompt_tokens"), ttft=a["ttft"], first_sentence=a["first_sentence"], total=a["total"]) if row["prompt_tokens"]: # Clamp: a server that under-reports tokens (e.g. counts only the # uncached part) would otherwise make each next prompt explode. chars_per_token = min(max(len(doc) / row["prompt_tokens"], 0.5), 4.0) # Same long prefix, a new question at the end: shows prompt-cache reuse. msgs2 = msgs[:1] + [{"role": "user", "content": doc + "\n\n위 글에서 가장 많이 나온 낱말 하나만 말해 줘."}] b = c.chat(msgs2, max_tokens=24, extra=NO_THINK) row.update(ttft_same_prefix=b["ttft"]) say(f" {row.get('prompt_tokens')} 토큰: 첫 토큰 {row['ttft']}s, 같은 앞부분 재사용 {row['ttft_same_prefix']}s") except Exception as e: r.error(f"context {target}", e) rows.append(row) r["context_ttft"] = rows def test_decode(r: StageResult, c: ChatClient, key: str = "decode") -> None: say(" - 생성 속도 (tok/s)") runs = [] for i in range(3): try: a = c.chat([{"role": "user", "content": f"({i}) 한국의 사계절을 각각 세 문장씩 자세히 설명해 줘."}], max_tokens=320, extra=NO_THINK) runs.append({k: a.get(k) for k in ("ttft", "total", "decode_tok_s", "usage", "server_timings")}) except Exception as e: r.error(f"{key} {i}", e) r[key] = {"runs": runs, "decode_tok_s": stats(x.get("decode_tok_s") for x in runs), "ttft": stats(x.get("ttft") for x in runs)} say(f" 중앙값 {r[key]['decode_tok_s'].get('median')} tok/s") try: code, body = http_json("GET", c.base + "/status", key=c.key, timeout=10) if code == 200 and isinstance(body, dict): r[f"{key}_server_last_timings"] = body.get("last_timings") except Exception: pass def test_thinking_default(r: StageResult, c: ChatClient) -> None: """Does the model think by default, and what does that cost the first answer token?""" try: a = c.chat([{"role": "user", "content": "자비스, 오늘 저녁 메뉴 하나만 추천해 줘."}], max_tokens=400) r["thinking_default"] = {k: a.get(k) for k in ("ttft", "first_reasoning", "reasoning_chars", "total", "text")} except Exception as e: r.error("thinking_default", e) # -------------------------------------------------------------------------- # Tool calling TOOLS = [ {"type": "function", "function": { "name": "get_weather", "description": "지정한 도시의 현재 날씨를 가져온다.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "도시 이름"}}, "required": ["city"]}}}, {"type": "function", "function": { "name": "set_timer", "description": "몇 분 뒤에 알리는 타이머를 맞춘다.", "parameters": {"type": "object", "properties": {"minutes": {"type": "integer"}}, "required": ["minutes"]}}}, {"type": "function", "function": { "name": "view_screen", "description": "사용자 PC의 화면을 캡처해서 본다.", "parameters": {"type": "object", "properties": { "monitor": {"type": "integer", "description": "모니터 번호 (1부터)"}, "quality": {"type": "string", "enum": ["fast", "normal", "detail"]}}, "required": ["monitor"]}}}, ] TOOL_CASES = [ ("서울 날씨 어때?", "get_weather", lambda a: "서울" in str(a.get("city", "")) or "Seoul" in str(a.get("city", ""))), ("5분 타이머 맞춰 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 5), ("두 번째 모니터에 뭐 떠 있어?", "view_screen", lambda a: int(a.get("monitor", 0)) == 2), ("안녕, 오늘 기분 어때?", None, None), ] def test_tools(r: StageResult, c: ChatClient) -> None: say(" - 도구 호출") rows = [] for prompt, want, check in TOOL_CASES: for i in range(3): row: dict[str, Any] = {"prompt": prompt, "expected": want, "try": i + 1} try: a = c.chat([{"role": "system", "content": VOICE_SYSTEM}, {"role": "user", "content": prompt}], max_tokens=200, tools=TOOLS, extra=NO_THINK) calls = a["tool_calls"] row.update(total=a["total"], first_tool=a["first_tool"], ttft=a["ttft"], finish=a["finish"], calls=calls, text=a["text"][:200]) if want is None: row["ok"] = not calls else: ok = bool(calls) and calls[0]["name"] == want if ok: try: ok = bool(check(json.loads(calls[0]["arguments"] or "{}"))) except Exception: ok = False row["ok"] = ok except Exception as e: row["error"] = str(e)[:300] row["ok"] = False rows.append(row) ok = sum(1 for x in rows if x.get("ok")) r["tools"] = {"cases": rows, "ok": ok, "total": len(rows)} say(f" 성공 {ok}/{len(rows)}") # Round trip: give the tool result back and read the final answer. try: msgs = [{"role": "system", "content": VOICE_SYSTEM}, {"role": "user", "content": "서울 날씨 어때?"}] a = c.chat(msgs, max_tokens=200, tools=TOOLS, extra=NO_THINK) if a["tool_calls"]: call = a["tool_calls"][0] msgs += [{"role": "assistant", "content": "", "tool_calls": [ {"id": "call_1", "type": "function", "function": {"name": call["name"], "arguments": call["arguments"]}}]}, {"role": "tool", "tool_call_id": "call_1", "content": json.dumps({"city": "서울", "temp_c": 13, "sky": "맑음"}, ensure_ascii=False)}] b = c.chat(msgs, max_tokens=120, tools=TOOLS, extra=NO_THINK) r["tools_round_trip"] = {"ttft": b["ttft"], "first_sentence": b["first_sentence"], "text": b["text"], "uses_result": "13" in b["text"]} except Exception as e: r.error("tools round trip", e) # -------------------------------------------------------------------------- # Pictures CODE_BIG = "4827" CODE_SMALL = "QX-7316" def _font(size: int): from PIL import ImageFont for p in (r"C:\Windows\Fonts\malgun.ttf", r"C:\Windows\Fonts\malgunbd.ttf", "/usr/share/fonts/truetype/nanum/NanumGothic.ttf", "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc"): if os.path.exists(p): return ImageFont.truetype(p, size) return ImageFont.load_default(size=size) def make_screen(seed: int = 0) -> "Any": """A synthetic 4K 'desktop' with a big number, a small code and Korean text. ``seed`` changes a few pixels' worth of content so a server cannot answer a repeated try from its picture cache. """ from PIL import Image, ImageDraw img = Image.new("RGB", (3840, 2160), (32, 56, 92)) d = ImageDraw.Draw(img) d.rectangle([200, 160, 2400, 1500], fill=(245, 245, 245)) d.rectangle([200, 160, 2400, 230], fill=(220, 220, 220)) d.text((230, 172), "메모장 - 오늘 할 일.txt", font=_font(40), fill=(20, 20, 20)) lines = ["1. 오후 세 시 회의 자료 정리", "2. 저녁 장보기: 두부, 계란, 대파", "3. 디스코드 서버 공지 올리기"] for i, line in enumerate(lines): d.text((260, 300 + i * 90), line, font=_font(56), fill=(30, 30, 30)) d.rectangle([2700, 300, 3500, 800], outline=(220, 30, 30), width=14) d.text((2830, 420), CODE_BIG, font=_font(220), fill=(220, 30, 30)) d.text((260, 1400), f"확인 코드: {CODE_SMALL}", font=_font(26), fill=(90, 90, 90)) d.rectangle([0, 2080, 3840, 2160], fill=(20, 20, 20)) d.text((3300, 2095), f"오후 {seed % 12 + 1}:{seed % 60:02d}", font=_font(44), fill=(230, 230, 230)) return img def _jpeg_b64(img: Any, long_side: int) -> tuple[str, float, tuple[int, int]]: t0 = time.perf_counter() w, h = img.size s = long_side / max(w, h) im = img.resize((max(1, round(w * s)), max(1, round(h * s)))) if s < 1 else img buf = io.BytesIO() im.save(buf, "JPEG", quality=85) return base64.b64encode(buf.getvalue()).decode(), time.perf_counter() - t0, im.size def picture_parts(img: Any, tier: str) -> tuple[list[dict], float, list]: if tier == "detail": w, h = img.size tiles = [img.crop((x, y, x + w // 2, y + h // 2)) for y in (0, h // 2) for x in (0, w // 2)] enc = [_jpeg_b64(t, 1280) for t in tiles] else: enc = [_jpeg_b64(img, {"fast": 768, "normal": 1280}[tier])] parts = [{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b}"}} for b, _, _ in enc] return parts, sum(t for _, t, _ in enc), [s for _, _, s in enc] def test_images(r: StageResult, c: ChatClient, key: str = "images") -> None: say(" - 이미지 읽기 (화질 단계별)") rows = [] question = ("이 화면에서 빨간 상자 안의 숫자와, 작은 글씨로 적힌 '확인 코드'를 읽어 줘. " "모르면 모른다고 해. 형식: 숫자=..., 코드=...") seed = random.randint(0, 10**6) for tier in ("fast", "normal", "detail"): for i in range(3): seed += 1 # The first picture after a start may include starting the picture reader. row: dict[str, Any] = {"tier": tier, "try": i + 1} try: parts, prep, sizes = picture_parts(make_screen(seed), tier) msgs = [{"role": "user", "content": parts + [{"type": "text", "text": question}]}] a = c.chat(msgs, max_tokens=60, extra=NO_THINK) row.update(prep_s=round(prep, 3), sizes=sizes, ttft=a["ttft"], total=a["total"], prompt_tokens=(a["usage"] or {}).get("prompt_tokens"), text=a["text"][:200], big_ok=CODE_BIG in a["text"], small_ok=CODE_SMALL.replace("-", "") in a["text"].replace("-", "").replace(" ", "").upper()) say(f" {tier}: 첫 토큰 {row['ttft']}s, 큰 숫자 {'O' if row['big_ok'] else 'X'}, " f"작은 코드 {'O' if row['small_ok'] else 'X'}") except Exception as e: row["error"] = str(e)[:300] say(f" {tier}: 실패 ({row['error'][:120]})") rows.append(row) r[key] = rows r[key + "_summary"] = {t: {"ttft_after_first": stats(x.get("ttft") for x in rows if x["tier"] == t and x["try"] > 1), "big_ok": sum(1 for x in rows if x["tier"] == t and x.get("big_ok")), "small_ok": sum(1 for x in rows if x["tier"] == t and x.get("small_ok")), "tries": sum(1 for x in rows if x["tier"] == t)} for t in ("fast", "normal", "detail")} # -------------------------------------------------------------------------- # Korean quality: answers are saved for reading KOREAN_PROMPTS = [ ("voice", "자비스, 오늘 비 온다는데 우산 챙겨야 할까?"), ("voice", "자비스, 라면 맛있게 끓이는 방법 알려 줘."), ("voice", "자비스, 아까 내가 무슨 노래 틀어 달라고 했지? (기억이 없으면 솔직하게 말해 줘)"), ("voice", "자비스, 3 곱하기 17 더하기 9는 얼마야?"), ("plain", "존댓말과 반말의 차이를 초등학생에게 설명하듯 세 문장으로 말해 줘."), ("plain", "다음 문장을 자연스러운 한국어로 고쳐 줘: '나는 어제 친구를 만나는 것을 했다 그리고 영화를 보았다.'"), ("plain", "\"내일 오후 2시에 강남역에서 민수랑 만나기로 했어\"에서 시간, 장소, 사람을 JSON으로 뽑아 줘."), ("plain", "Translate to natural Korean: 'I'll be right back, just grabbing a coffee.'"), ] def test_korean(r: StageResult, c: ChatClient) -> None: say(" - 한국어 답변 모음") rows = [] for kind, prompt in KOREAN_PROMPTS: msgs = ([{"role": "system", "content": VOICE_SYSTEM}] if kind == "voice" else []) + \ [{"role": "user", "content": prompt}] try: a = c.chat(msgs, max_tokens=300, extra=NO_THINK) rows.append({"kind": kind, "prompt": prompt, "answer": a["text"], "ttft": a["ttft"], "first_sentence": a["first_sentence"], "total": a["total"]}) except Exception as e: rows.append({"kind": kind, "prompt": prompt, "error": str(e)[:300]}) r["korean"] = rows r["voice_first_sentence"] = stats(x.get("first_sentence") for x in rows if x["kind"] == "voice") # -------------------------------------------------------------------------- # Several requests at once def test_parallel(r: StageResult, c: ChatClient, n_list=(2, 4)) -> None: say(" - 동시 요청") out = {} for n in n_list: results: list[dict] = [{} for _ in range(n)] def one(i: int) -> None: try: a = c.chat([{"role": "user", "content": f"({i}) 좋아하는 계절 하나를 골라 이유를 다섯 문장으로 말해 줘."}], max_tokens=160, extra=NO_THINK) results[i] = {k: a.get(k) for k in ("ttft", "total", "decode_tok_s")} results[i]["completion_tokens"] = (a.get("usage") or {}).get("completion_tokens") except Exception as e: results[i] = {"error": str(e)[:200]} t0 = time.perf_counter() th = [threading.Thread(target=one, args=(i,)) for i in range(n)] for t in th: t.start() for t in th: t.join() wall = time.perf_counter() - t0 toks = sum(x.get("completion_tokens") or 0 for x in results) out[str(n)] = {"wall_s": round(wall, 2), "sum_tokens": toks, "aggregate_tok_s": round(toks / wall, 1), "ttft": stats(x.get("ttft") for x in results), "requests": results} say(f" {n}개 동시: 전체 {out[str(n)]['aggregate_tok_s']} tok/s, " f"첫 토큰 최대 {out[str(n)]['ttft'].get('max')}s") r["parallel"] = out # -------------------------------------------------------------------------- # Game mode (a): give VRAM back with POST /v1/vram and take it again def test_vram_shrink(r: StageResult, c: ChatClient, reserve_mib: int = 8000) -> None: say(f" - 게임 모드 (a): VRAM {reserve_mib} MiB 비우기") out: dict[str, Any] = {"reserve_mib": reserve_mib, "gpu_before": gpu_memory_mib()} root = c.base[:-3] if c.base.endswith("/v1") else c.base try: t0 = time.perf_counter() code, body = http_json("POST", root + "/v1/vram", {"reserve_mib": reserve_mib}, timeout=400, key=c.key) out["shrink"] = {"http": code, "seconds": round(time.perf_counter() - t0, 3), "body": body} out["gpu_after_shrink"] = gpu_memory_mib() if code == 200: a = c.chat([{"role": "user", "content": "한국의 사계절을 각각 세 문장씩 설명해 줘."}], max_tokens=200, extra=NO_THINK) out["decode_tok_s_shrunk"] = a.get("decode_tok_s") out["ttft_shrunk"] = a.get("ttft") t0 = time.perf_counter() code2, body2 = http_json("POST", root + "/v1/vram", {"reserve_mib": None}, timeout=400, key=c.key) out["grow"] = {"http": code2, "seconds": round(time.perf_counter() - t0, 3), "body": body2} out["gpu_after_grow"] = gpu_memory_mib() say(f" 비우기 HTTP {code} ({out['shrink']['seconds']}s), 되돌리기 HTTP {code2} ({out['grow']['seconds']}s), " f"비운 상태 속도 {out.get('decode_tok_s_shrunk')} tok/s") except Exception as e: r.error("vram shrink", e) r["vram_shrink"] = out def quick_bench(c: ChatClient, context_tokens: int = 8192) -> dict[str, Any]: """The numbers a placement change moves, in about half a minute: output speed, the first spoken sentence and the first token after a long context. Fail-open: an error is kept by name and the next measurement runs.""" out: dict[str, Any] = {"errors": []} runs = [] for i in range(3): try: a = c.chat([{"role": "user", "content": f"({i}) 한국의 사계절을 각각 세 문장씩 자세히 설명해 줘."}], max_tokens=320, extra=NO_THINK) runs.append(a) except Exception as e: out["errors"].append(f"decode: {type(e).__name__}") out["decode_tok_s"] = stats(x.get("decode_tok_s") for x in runs) voice = [] for kind, prompt in KOREAN_PROMPTS: if kind != "voice": continue try: a = c.chat([{"role": "system", "content": VOICE_SYSTEM}, {"role": "user", "content": prompt}], max_tokens=120, extra=NO_THINK) voice.append(a.get("first_sentence")) except Exception as e: out["errors"].append(f"voice: {type(e).__name__}") out["voice_first_sentence"] = stats(voice) seed = random.randint(0, 10**9) doc = f"[문서 {seed}]\n" + filler(int(context_tokens * 1.5), seed) try: a = c.chat([{"role": "system", "content": VOICE_SYSTEM}, {"role": "user", "content": doc + "\n\n위 글의 주제를 한 문장으로 말해 줘."}], max_tokens=48, extra=NO_THINK) out["context"] = {"prompt_tokens": (a.get("usage") or {}).get("prompt_tokens"), "ttft": a.get("ttft"), "first_sentence": a.get("first_sentence")} except Exception as e: out["errors"].append(f"context: {type(e).__name__}") return out def warmup(r: StageResult, c: ChatClient) -> None: """One throwaway request so model loading is not counted as answer time.""" try: a = c.chat([{"role": "user", "content": "안녕"}], max_tokens=8, extra=NO_THINK) r["warmup_total_s"] = a["total"] except Exception as e: r.error("warmup", e) def bench_single(r: StageResult, c: ChatClient, ctx_limit: int) -> None: """The tests that need one request at a time.""" for fn in (lambda: warmup(r, c), lambda: test_decode(r, c), lambda: test_thinking_default(r, c), lambda: test_context(r, c, ctx_limit), lambda: test_tools(r, c), lambda: test_images(r, c), lambda: test_korean(r, c)): try: fn() except Exception as e: r.error("bench", e) r.save()