fix(m0): bound context-test prompt size when tokens are under-reported
The context test re-estimated characters per token from the server's prompt_tokens after every request. A server that reports a tiny count made each next prompt grow by orders of magnitude, so the kit tests reached 8 GB and were killed by the host memory guard. Clamp the estimate to 0.5-4 characters per token. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -61,7 +61,9 @@ def test_context(r: StageResult, c: ChatClient, ctx_limit: int) -> None:
|
||||
row.update(prompt_tokens=(a["usage"] or {}).get("prompt_tokens"), ttft=a["ttft"],
|
||||
first_sentence=a["first_sentence"], total=a["total"])
|
||||
if row["prompt_tokens"]:
|
||||
chars_per_token = len(doc) / row["prompt_tokens"]
|
||||
# Clamp: a server that under-reports tokens (e.g. counts only the
|
||||
# uncached part) would otherwise make each next prompt explode.
|
||||
chars_per_token = min(max(len(doc) / row["prompt_tokens"], 0.5), 4.0)
|
||||
# Same long prefix, a new question at the end: shows prompt-cache reuse.
|
||||
msgs2 = msgs[:1] + [{"role": "user", "content": doc + "\n\n위 글에서 가장 많이 나온 낱말 하나만 말해 줘."}]
|
||||
b = c.chat(msgs2, max_tokens=24, extra=NO_THINK)
|
||||
|
||||
@@ -16,7 +16,7 @@ import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0"))
|
||||
|
||||
from m0lib import common, llm # noqa: E402
|
||||
from m0lib import common, llm, llm_bench # noqa: E402
|
||||
from m0lib.llm_client import ChatClient # noqa: E402
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
@@ -62,8 +62,12 @@ class _FakeOpenAI(BaseHTTPRequestHandler):
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
request_sizes: list[int] = []
|
||||
|
||||
def do_POST(self):
|
||||
self.rfile.read(int(self.headers.get("Content-Length", 0)))
|
||||
n = int(self.headers.get("Content-Length", 0))
|
||||
self.rfile.read(n)
|
||||
_FakeOpenAI.request_sizes.append(n)
|
||||
chunks = [
|
||||
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"name": "get_weather", "arguments": ""}}]}}]},
|
||||
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '{"city": '}}]}}]},
|
||||
@@ -102,6 +106,18 @@ def test_chat_client_assembles_streamed_tool_call_and_timings(fake_server):
|
||||
assert a["first_tool"] <= a["ttft"] <= a["first_sentence"] <= a["total"]
|
||||
|
||||
|
||||
def test_context_prompts_stay_bounded_when_server_under_reports_tokens(tmp_path, fake_server):
|
||||
# The fake server always reports 12 prompt tokens. The context test must not
|
||||
# trust that blindly and grow each next prompt without limit (it used to
|
||||
# reach gigabytes and exhaust RAM).
|
||||
_FakeOpenAI.request_sizes.clear()
|
||||
largest_target = 24576
|
||||
llm_bench.test_context(common.StageResult("t", tmp_path / "t.json"), ChatClient(fake_server), 10**6)
|
||||
assert _FakeOpenAI.request_sizes
|
||||
# Generous bound: 8 characters per token, 4 UTF-8 bytes per character, x2 for JSON escaping.
|
||||
assert max(_FakeOpenAI.request_sizes) < largest_target * 8 * 4 * 2
|
||||
|
||||
|
||||
# --- LLM stage: a failed start must not count as measured -------------------
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user