diff --git a/scripts/m0/m0lib/llm_bench.py b/scripts/m0/m0lib/llm_bench.py index 4d70940..c37ae86 100644 --- a/scripts/m0/m0lib/llm_bench.py +++ b/scripts/m0/m0lib/llm_bench.py @@ -61,7 +61,9 @@ def test_context(r: StageResult, c: ChatClient, ctx_limit: int) -> None: row.update(prompt_tokens=(a["usage"] or {}).get("prompt_tokens"), ttft=a["ttft"], first_sentence=a["first_sentence"], total=a["total"]) if row["prompt_tokens"]: - chars_per_token = len(doc) / row["prompt_tokens"] + # Clamp: a server that under-reports tokens (e.g. counts only the + # uncached part) would otherwise make each next prompt explode. + chars_per_token = min(max(len(doc) / row["prompt_tokens"], 0.5), 4.0) # Same long prefix, a new question at the end: shows prompt-cache reuse. msgs2 = msgs[:1] + [{"role": "user", "content": doc + "\n\n위 글에서 가장 많이 나온 낱말 하나만 말해 줘."}] b = c.chat(msgs2, max_tokens=24, extra=NO_THINK) diff --git a/tests/test_m0_kit.py b/tests/test_m0_kit.py index 7ae05d8..e6fa1aa 100644 --- a/tests/test_m0_kit.py +++ b/tests/test_m0_kit.py @@ -16,7 +16,7 @@ import pytest sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0")) -from m0lib import common, llm # noqa: E402 +from m0lib import common, llm, llm_bench # noqa: E402 from m0lib.llm_client import ChatClient # noqa: E402 pytestmark = pytest.mark.unit @@ -62,8 +62,12 @@ class _FakeOpenAI(BaseHTTPRequestHandler): self.end_headers() self.wfile.write(body) + request_sizes: list[int] = [] + def do_POST(self): - self.rfile.read(int(self.headers.get("Content-Length", 0))) + n = int(self.headers.get("Content-Length", 0)) + self.rfile.read(n) + _FakeOpenAI.request_sizes.append(n) chunks = [ {"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"name": "get_weather", "arguments": ""}}]}}]}, {"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '{"city": '}}]}}]}, @@ -102,6 +106,18 @@ def test_chat_client_assembles_streamed_tool_call_and_timings(fake_server): assert a["first_tool"] <= a["ttft"] <= a["first_sentence"] <= a["total"] +def test_context_prompts_stay_bounded_when_server_under_reports_tokens(tmp_path, fake_server): + # The fake server always reports 12 prompt tokens. The context test must not + # trust that blindly and grow each next prompt without limit (it used to + # reach gigabytes and exhaust RAM). + _FakeOpenAI.request_sizes.clear() + largest_target = 24576 + llm_bench.test_context(common.StageResult("t", tmp_path / "t.json"), ChatClient(fake_server), 10**6) + assert _FakeOpenAI.request_sizes + # Generous bound: 8 characters per token, 4 UTF-8 bytes per character, x2 for JSON escaping. + assert max(_FakeOpenAI.request_sizes) < largest_target * 8 * 4 * 2 + + # --- LLM stage: a failed start must not count as measured -------------------