feat: per-call LLM timing, speaker ID, cancel captures on leave
- llm.py: log each Ollama call's caller + total/load/prompt/gen durations so a slow voice turn is attributable to a specific internal call (router/enrichment/digest/main); a RELOAD marker flags cold reloads. - voice.ts: track in-flight Opus captures and abort them on session destroy(); drop any utterance that finishes after the user left, so no trailing post-leave VAD turns are reported. - userbot.ts: show the speaker's Discord user ID on each transcript line (answered and dropped) so it's clear whose audio produced the turn. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -3,9 +3,19 @@
|
||||
from __future__ import annotations
|
||||
from typing import Optional, Any, Dict, List, Generator, Callable
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
import json
|
||||
|
||||
|
||||
def _caller_name() -> str:
|
||||
"""Best-effort name of the function that invoked the LLM wrapper, used to
|
||||
label per-call timing (router / enrichment / digest / main)."""
|
||||
try:
|
||||
return sys._getframe(2).f_code.co_name
|
||||
except Exception:
|
||||
return "?"
|
||||
|
||||
from .debug import debug_log
|
||||
|
||||
|
||||
@@ -25,6 +35,33 @@ class ToolsNotSupportedError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _log_ollama_timing(data: Dict[str, Any], num_ctx: int, caller: str) -> None:
|
||||
"""Emit a one-line per-call latency breakdown so a slow voice turn can be
|
||||
attributed to a specific internal LLM call (router / enrichment / digest /
|
||||
main) instead of just a total. ``load_duration`` > 0 means the model was
|
||||
cold-reloaded for this call — the single most expensive thing to avoid.
|
||||
"""
|
||||
if not isinstance(data, dict):
|
||||
return
|
||||
try:
|
||||
ns = 1e9
|
||||
total = data.get("total_duration", 0) / ns
|
||||
load = data.get("load_duration", 0) / ns
|
||||
peval = data.get("prompt_eval_duration", 0) / ns
|
||||
pcount = data.get("prompt_eval_count")
|
||||
gen = data.get("eval_duration", 0) / ns
|
||||
gcount = data.get("eval_count")
|
||||
reload_flag = " RELOAD" if load > 0.5 else ""
|
||||
print(
|
||||
f" ⏱️ llm[{caller}] ctx={num_ctx} total={total:.2f}s "
|
||||
f"load={load:.2f}s{reload_flag} prompt={peval:.2f}s({pcount}t) "
|
||||
f"gen={gen:.2f}s({gcount}t)",
|
||||
flush=True,
|
||||
)
|
||||
except Exception: # pragma: no cover - logging must never break a reply
|
||||
pass
|
||||
|
||||
|
||||
def call_llm_direct(base_url: str, chat_model: str, system_prompt: str, user_content: str, timeout_sec: float = 10.0, thinking: bool = False, num_ctx: int = OLLAMA_NUM_CTX, temperature: Optional[float] = None) -> Optional[str]:
|
||||
"""Direct LLM call without temporal context, location, or other ask_coach features.
|
||||
|
||||
@@ -62,10 +99,12 @@ def call_llm_direct(base_url: str, chat_model: str, system_prompt: str, user_con
|
||||
"keep_alive": "30m",
|
||||
}
|
||||
|
||||
caller = _caller_name()
|
||||
try:
|
||||
with requests.post(f"{base_url.rstrip('/')}/api/chat", json=payload, timeout=timeout_sec) as resp:
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
_log_ollama_timing(data, num_ctx, caller)
|
||||
|
||||
if isinstance(data, dict):
|
||||
content = extract_text_from_response(data)
|
||||
@@ -233,10 +272,12 @@ def chat_with_messages(
|
||||
if tools and isinstance(tools, list) and len(tools) > 0:
|
||||
payload["tools"] = tools
|
||||
|
||||
caller = _caller_name()
|
||||
try:
|
||||
with requests.post(f"{base_url.rstrip('/')}/api/chat", json=payload, timeout=timeout_sec) as resp:
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
_log_ollama_timing(data, OLLAMA_NUM_CTX, caller)
|
||||
if isinstance(data, dict):
|
||||
return data
|
||||
except requests.exceptions.Timeout:
|
||||
|
||||
Reference in New Issue
Block a user