fix(m0): bound context-test prompt size when tokens are under-reported

The context test re-estimated characters per token from the server's
prompt_tokens after every request. A server that reports a tiny count
made each next prompt grow by orders of magnitude, so the kit tests
reached 8 GB and were killed by the host memory guard. Clamp the
estimate to 0.5-4 characters per token.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-09 17:20:01 +09:00
parent 5762a22a67
commit ff9b868e62
2 changed files with 21 additions and 3 deletions

View File

@@ -61,7 +61,9 @@ def test_context(r: StageResult, c: ChatClient, ctx_limit: int) -> None:
row.update(prompt_tokens=(a["usage"] or {}).get("prompt_tokens"), ttft=a["ttft"],
first_sentence=a["first_sentence"], total=a["total"])
if row["prompt_tokens"]:
chars_per_token = len(doc) / row["prompt_tokens"]
# Clamp: a server that under-reports tokens (e.g. counts only the
# uncached part) would otherwise make each next prompt explode.
chars_per_token = min(max(len(doc) / row["prompt_tokens"], 0.5), 4.0)
# Same long prefix, a new question at the end: shows prompt-cache reuse.
msgs2 = msgs[:1] + [{"role": "user", "content": doc + "\n\n위 글에서 가장 많이 나온 낱말 하나만 말해 줘."}]
b = c.chat(msgs2, max_tokens=24, extra=NO_THINK)

View File

@@ -16,7 +16,7 @@ import pytest
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0"))
from m0lib import common, llm # noqa: E402
from m0lib import common, llm, llm_bench # noqa: E402
from m0lib.llm_client import ChatClient # noqa: E402
pytestmark = pytest.mark.unit
@@ -62,8 +62,12 @@ class _FakeOpenAI(BaseHTTPRequestHandler):
self.end_headers()
self.wfile.write(body)
request_sizes: list[int] = []
def do_POST(self):
self.rfile.read(int(self.headers.get("Content-Length", 0)))
n = int(self.headers.get("Content-Length", 0))
self.rfile.read(n)
_FakeOpenAI.request_sizes.append(n)
chunks = [
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"name": "get_weather", "arguments": ""}}]}}]},
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '{"city": '}}]}}]},
@@ -102,6 +106,18 @@ def test_chat_client_assembles_streamed_tool_call_and_timings(fake_server):
assert a["first_tool"] <= a["ttft"] <= a["first_sentence"] <= a["total"]
def test_context_prompts_stay_bounded_when_server_under_reports_tokens(tmp_path, fake_server):
# The fake server always reports 12 prompt tokens. The context test must not
# trust that blindly and grow each next prompt without limit (it used to
# reach gigabytes and exhaust RAM).
_FakeOpenAI.request_sizes.clear()
largest_target = 24576
llm_bench.test_context(common.StageResult("t", tmp_path / "t.json"), ChatClient(fake_server), 10**6)
assert _FakeOpenAI.request_sizes
# Generous bound: 8 characters per token, 4 UTF-8 bytes per character, x2 for JSON escaping.
assert max(_FakeOpenAI.request_sizes) < largest_target * 8 * 4 * 2
# --- LLM stage: a failed start must not count as measured -------------------