"""Minimal OpenAI-compatible chat client with timing (standard library only).""" from __future__ import annotations import json import re import time import urllib.error import urllib.request from typing import Any # A sentence ends at . ? ! or a CJK full stop, followed by space/end, or at a newline. _SENTENCE_END = re.compile(r"([.?!。?!][\"')\]]*(\s|$))|\n") def http_json(method: str, url: str, body: Any = None, timeout: float = 30, key: str | None = None) -> tuple[int, Any]: data = json.dumps(body).encode("utf-8") if body is not None else None req = urllib.request.Request(url, data=data, method=method) req.add_header("Content-Type", "application/json") if key: req.add_header("Authorization", f"Bearer {key}") try: with urllib.request.urlopen(req, timeout=timeout) as resp: raw = resp.read().decode("utf-8", "replace") return resp.status, (json.loads(raw) if raw.strip() else None) except urllib.error.HTTPError as e: raw = e.read().decode("utf-8", "replace") try: return e.code, json.loads(raw) except ValueError: return e.code, raw[:2000] class ChatClient: def __init__(self, base_url: str, model: str | None = None, key: str | None = None, extra: dict[str, Any] | None = None, timeout: float = 600): self.base = base_url.rstrip("/") self.key = key self.timeout = timeout self.extra = extra or {} self.model = model or self.first_model() def first_model(self) -> str: code, body = http_json("GET", f"{self.base}/models", key=self.key) if code != 200 or not isinstance(body, dict) or not body.get("data"): raise RuntimeError(f"/models answered {code}: {str(body)[:300]}") return body["data"][0]["id"] def chat(self, messages: list[dict], *, max_tokens: int = 256, tools: list | None = None, temperature: float = 0.0, stream: bool = True, extra: dict | None = None) -> dict[str, Any]: """Send one chat request and return timings plus the answer. Times are seconds from sending the request: ``ttft`` (first answer text), ``first_reasoning`` (first thinking text, if any), ``first_sentence`` (first complete sentence of the answer), ``total``. """ body: dict[str, Any] = {"model": self.model, "messages": messages, "max_tokens": max_tokens, "temperature": temperature, "stream": stream} if stream: body["stream_options"] = {"include_usage": True} if tools: body["tools"] = tools body.update(self.extra) if extra: body.update(extra) req = urllib.request.Request(f"{self.base}/chat/completions", data=json.dumps(body).encode("utf-8"), method="POST") req.add_header("Content-Type", "application/json") if self.key: req.add_header("Authorization", f"Bearer {self.key}") out: dict[str, Any] = {"ttft": None, "first_reasoning": None, "first_sentence": None, "first_tool": None} content, reasoning = [], [] tool_calls: dict[int, dict[str, Any]] = {} usage, finish, last_chunk = None, None, None t0 = time.perf_counter() try: resp = urllib.request.urlopen(req, timeout=self.timeout) except urllib.error.HTTPError as e: raise RuntimeError(f"HTTP {e.code}: {e.read().decode('utf-8', 'replace')[:500]}") from None with resp: if not stream: body_out = json.loads(resp.read().decode("utf-8")) out["total"] = time.perf_counter() - t0 msg = body_out["choices"][0]["message"] content = [msg.get("content") or ""] reasoning = [msg.get("reasoning_content") or ""] for i, tc in enumerate(msg.get("tool_calls") or []): tool_calls[i] = {"name": tc["function"]["name"], "arguments": tc["function"].get("arguments", "")} usage, finish, last_chunk = body_out.get("usage"), body_out["choices"][0].get("finish_reason"), body_out else: for raw in resp: line = raw.decode("utf-8", "replace").strip() if not line.startswith("data:"): continue payload = line[5:].strip() if payload == "[DONE]": break try: chunk = json.loads(payload) except ValueError: continue last_chunk = chunk if chunk.get("usage"): usage = chunk["usage"] for ch in chunk.get("choices") or []: d = ch.get("delta") or {} now = time.perf_counter() - t0 if d.get("reasoning_content"): if out["first_reasoning"] is None: out["first_reasoning"] = now reasoning.append(d["reasoning_content"]) if d.get("content"): if out["ttft"] is None: out["ttft"] = now content.append(d["content"]) if out["first_sentence"] is None and _SENTENCE_END.search("".join(content)): out["first_sentence"] = now for tc in d.get("tool_calls") or []: if out["first_tool"] is None: out["first_tool"] = now slot = tool_calls.setdefault(tc.get("index", 0), {"name": "", "arguments": ""}) fn = tc.get("function") or {} if fn.get("name"): slot["name"] = fn["name"] if fn.get("arguments"): slot["arguments"] += fn["arguments"] if ch.get("finish_reason"): finish = ch["finish_reason"] out["total"] = time.perf_counter() - t0 text = "".join(content) if out["first_sentence"] is None and text: out["first_sentence"] = out["total"] out.update({"text": text, "reasoning_chars": len("".join(reasoning)), "finish": finish, "usage": usage, "tool_calls": [tool_calls[k] for k in sorted(tool_calls)]}) comp = (usage or {}).get("completion_tokens") first = out["ttft"] if out["ttft"] is not None else out["first_reasoning"] if comp and first is not None and out["total"] > first and comp > 1: out["decode_tok_s"] = round((comp - 1) / (out["total"] - first), 1) if last_chunk and isinstance(last_chunk, dict): for k in ("timings", "strata"): if k in last_chunk: out[f"server_{k}"] = last_chunk[k] for k in ("ttft", "first_reasoning", "first_sentence", "first_tool", "total"): if out.get(k) is not None: out[k] = round(out[k], 3) return out