feat(voice_worker): a short voice prompt and one judgement before each v2 answer
- compact_prompt: the v2 persona in a tenth of v1's words, answers as short as possible, the full tool guidance only on turns with a data tool. A greeting's reply prompt went from about 4,500 to 1,400 tokens on Strata. - intent_judge: the tool router judges whether the turn needs tools; the planner runs only when it does. prepare_intent runs the pick ahead on a partial transcript (worker message prepare), and the reply reuses it. - Without stream_browser the screen-share tools are not offered. - The time context carries the 12-hour form (10:37 was read as pm). - A <tool_call> block written as the reply is parsed as a call, not spoken. Evals (Qwen3.6 on Strata, EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1): 221 passed, 4 failed (known failures); new evals/test_voice_reply_length.py (v2 answers 234 chars against v1's 416) and test_reply_language_default.py. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -11,6 +11,8 @@ from typing import Optional, TYPE_CHECKING
|
||||
from ..utils.redact import redact
|
||||
from ..system_prompt import (
|
||||
build_system_prompt,
|
||||
build_voice_persona,
|
||||
COMPACT_ACTION_RULE,
|
||||
load_agent_instructions,
|
||||
reply_language_directive,
|
||||
tool_honesty_reminder,
|
||||
@@ -321,6 +323,26 @@ def _extract_text_tool_call(content_field: str, known_names: set):
|
||||
return None, None, None
|
||||
content_field = content_field
|
||||
|
||||
# Form: `<tool_call>` block as text — Qwen's XML (`<function=name><parameter=key>value</parameter>`)
|
||||
# or Hermes JSON (`{"name": ..., "arguments": {...}}`). Field run 2026-11-12 (Qwen3.6 on Strata):
|
||||
# asked for a timer with no timer tool offered, the model wrote this block as its reply, which
|
||||
# TTS would read aloud. Any name is returned: one not on offer gets the "not available" answer.
|
||||
tc_block = re.search(r"<tool_call>\s*(.+?)\s*(?:</tool_call>|$)", content_field, re.DOTALL)
|
||||
if tc_block:
|
||||
body = tc_block.group(1)
|
||||
fn = re.match(r"<function=([^>\s]+)>(.*?)(?:</function>|$)", body, re.DOTALL)
|
||||
if fn:
|
||||
args = {k: v.strip() for k, v in re.findall(r"<parameter=([^>\s]+)>(.*?)</parameter>", fn.group(2), re.DOTALL)}
|
||||
return fn.group(1).strip(), args, f"call_{uuid.uuid4().hex[:8]}"
|
||||
try:
|
||||
data = json.loads(body)
|
||||
name = str(data.get("name", "")).strip() if isinstance(data, dict) else ""
|
||||
if name:
|
||||
args = data.get("arguments", {})
|
||||
return name, (args if isinstance(args, dict) else {}), f"call_{uuid.uuid4().hex[:8]}"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Form: markdown fence
|
||||
fence_match = re.search(
|
||||
r"```tool_call\s*\n({.+?})\s*\n```",
|
||||
@@ -977,12 +999,95 @@ def _history_block(history: Any) -> str:
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _dialogue_context(recent_messages: list) -> str:
|
||||
"""The planner's DIALOGUE CONTEXT: the last few user/assistant lines, shortened."""
|
||||
lines: list[str] = []
|
||||
for m in (recent_messages or [])[-6:]:
|
||||
role = m.get("role", "")
|
||||
content = (m.get("content") or "").strip().replace("\n", " ")
|
||||
if role in ("user", "assistant") and content:
|
||||
lines.append(f"{role}: {content[:200]}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _recent_messages(cfg, dialogue_memory) -> list:
|
||||
if not (dialogue_memory and dialogue_memory.has_recent_messages()):
|
||||
return []
|
||||
if hasattr(dialogue_memory, "get_recent_turns_with_tools"):
|
||||
return dialogue_memory.get_recent_turns_with_tools(
|
||||
max_tool_turns=getattr(cfg, "tool_carryover_max_turns", 2),
|
||||
per_entry_chars=getattr(cfg, "tool_carryover_per_entry_chars", 1200),
|
||||
)
|
||||
return dialogue_memory.get_recent_messages()
|
||||
|
||||
|
||||
# The on-screen browser tools work only in screen-share mode (cfg.stream_browser); without it they
|
||||
# only refuse, and offered anyway the router picked them for unrelated asks (field run 2026-11-12,
|
||||
# v2: "피곤하다" routed to browseAndPlay).
|
||||
_SCREEN_SHARE_TOOLS = ("controlBrowser", "browseAndPlay", "setBroadcast")
|
||||
|
||||
|
||||
def _offered_builtins(cfg, chat_available: bool) -> dict:
|
||||
"""The built-in tools this turn can offer: chatChannels only with a chat backend
|
||||
(chat_channels.spec.md), the screen-share tools only in screen-share mode."""
|
||||
stream = getattr(cfg, "stream_browser", True)
|
||||
return {k: v for k, v in BUILTIN_TOOLS.items()
|
||||
if (k != "chatChannels" or chat_available) and (stream or k not in _SCREEN_SHARE_TOOLS)}
|
||||
|
||||
|
||||
def _route_tools(cfg, redacted: str, builtin_tools: dict, mcp_tools: dict, recent_messages: list):
|
||||
"""The tool router's pick for this query (``select_tools`` with the engine's settings)."""
|
||||
try:
|
||||
strategy = ToolSelectionStrategy(getattr(cfg, "tool_selection_strategy", "llm"))
|
||||
except ValueError:
|
||||
strategy = ToolSelectionStrategy.LLM
|
||||
return select_tools(
|
||||
query=redacted,
|
||||
builtin_tools=builtin_tools,
|
||||
mcp_tools=mcp_tools,
|
||||
strategy=strategy,
|
||||
llm_base_url=cfg.ollama_base_url,
|
||||
llm_model=resolve_tool_router_model(cfg),
|
||||
llm_timeout_sec=float(getattr(cfg, "llm_tools_timeout_sec", 8.0)),
|
||||
embed_model=getattr(cfg, "ollama_embed_model", "nomic-embed-text"),
|
||||
embed_timeout_sec=float(getattr(cfg, "llm_embed_timeout_sec", 10.0)),
|
||||
context_hint=_build_enrichment_context_hint(cfg, recent_messages),
|
||||
)
|
||||
|
||||
|
||||
def prepare_intent(cfg, text: str, dialogue_memory, chat_available: bool) -> Optional[list]:
|
||||
"""Run the intent judge (the tool router) for ``text`` ahead of its reply (v2: on a partial
|
||||
transcript while the user still speaks), with the inputs ``run_reply_engine(intent_judge=True)``
|
||||
would give it, so its pick can be passed back as ``prepared_tools``. None when the router gave
|
||||
up (its fall-open to the whole catalogue is not a pick). ``chat_available``: the reply will
|
||||
have a chat backend (the chatChannels tool is in the catalogue)."""
|
||||
mcp_tools: dict = {}
|
||||
if getattr(cfg, "mcps", {}):
|
||||
try:
|
||||
from ..tools.registry import get_cached_mcp_tools
|
||||
mcp_tools = get_cached_mcp_tools()
|
||||
except Exception:
|
||||
mcp_tools = {}
|
||||
builtin = _offered_builtins(cfg, chat_available)
|
||||
try:
|
||||
routed = _route_tools(cfg, redact(text), builtin, mcp_tools, _recent_messages(cfg, dialogue_memory))
|
||||
except Exception as exc: # pragma: no cover — defensive
|
||||
debug_log(f"intent judge (prepare) failed (non-fatal): {exc}", "planning")
|
||||
return None
|
||||
if routed is None or set(routed) >= set(builtin) | set(mcp_tools):
|
||||
return None
|
||||
return list(routed)
|
||||
|
||||
|
||||
def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
text: str, dialogue_memory: "DialogueMemory",
|
||||
language: Optional[str] = None,
|
||||
history: Optional[Any] = None,
|
||||
legacy_memory: bool = True,
|
||||
default_language: Optional[str] = None) -> Optional[str]:
|
||||
default_language: Optional[str] = None,
|
||||
compact_prompt: bool = False,
|
||||
intent_judge: bool = False,
|
||||
prepared_tools: Optional[list] = None) -> Optional[str]:
|
||||
"""
|
||||
Main entry point for reply generation.
|
||||
|
||||
@@ -1008,6 +1113,15 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
default_language: v2 only. The language to reply in unless the user
|
||||
explicitly asks for another one (e.g. "Korean"). An
|
||||
``output_language`` lock still wins. None: the user's language.
|
||||
compact_prompt: v2 only. The short voice persona and, on turns where
|
||||
no data tool was routed, one action rule instead of the full tool
|
||||
guidance (prompt length is reply latency on Strata).
|
||||
intent_judge: v2 only. One judgement before the reply: the tool
|
||||
router decides which tools the turn needs and the planner does
|
||||
not run (planner.spec.md, "Intent judge (v2)").
|
||||
prepared_tools: v2 only, with ``intent_judge``. The router's tools
|
||||
``prepare_intent`` already chose for this text (while the user was
|
||||
still speaking); the router is not run again.
|
||||
|
||||
Returns:
|
||||
Generated reply text or None
|
||||
@@ -1095,19 +1209,12 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
# (chat_channels.spec.md); otherwise it is not in the catalogue at all.
|
||||
from .turn_state import get_chat as _get_chat
|
||||
_chat_backend = _get_chat()
|
||||
_builtin_tools = {k: v for k, v in BUILTIN_TOOLS.items()
|
||||
if k != "chatChannels" or _chat_backend is not None}
|
||||
_builtin_tools = _offered_builtins(cfg, _chat_backend is not None)
|
||||
_all_builtin_names = list(_builtin_tools.keys())
|
||||
_all_mcp_names = list(mcp_tools.keys())
|
||||
_full_catalog_names = _all_builtin_names + _all_mcp_names
|
||||
|
||||
_dialogue_lines: list[str] = []
|
||||
for _m in (recent_messages or [])[-6:]:
|
||||
_role = _m.get("role", "")
|
||||
_content = (_m.get("content") or "").strip().replace("\n", " ")
|
||||
if _role in ("user", "assistant") and _content:
|
||||
_dialogue_lines.append(f"{_role}: {_content[:200]}")
|
||||
_dialogue_ctx = "\n".join(_dialogue_lines)
|
||||
_dialogue_ctx = _dialogue_context(recent_messages)
|
||||
|
||||
# Step 2a: Tool routing FIRST.
|
||||
#
|
||||
@@ -1138,22 +1245,15 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
dialogue_memory.hot_cache_get(_router_cache_key)
|
||||
if dialogue_memory and hasattr(dialogue_memory, "hot_cache_get") else None
|
||||
)
|
||||
if isinstance(_cached_routed, list):
|
||||
if intent_judge and prepared_tools is not None:
|
||||
# Intent judge (v2): the router's pick made while the user was still speaking.
|
||||
routed_tools = list(prepared_tools)
|
||||
debug_log(f"intent judge: prepared pick used ({len(routed_tools)} tool(s))", "planning")
|
||||
elif isinstance(_cached_routed, list):
|
||||
routed_tools = list(_cached_routed)
|
||||
debug_log("tool router served from hot-window cache", "planning")
|
||||
else:
|
||||
routed_tools = select_tools(
|
||||
query=redacted,
|
||||
builtin_tools=_builtin_tools,
|
||||
mcp_tools=mcp_tools,
|
||||
strategy=strategy,
|
||||
llm_base_url=cfg.ollama_base_url,
|
||||
llm_model=resolve_tool_router_model(cfg),
|
||||
llm_timeout_sec=float(getattr(cfg, "llm_tools_timeout_sec", 8.0)),
|
||||
embed_model=getattr(cfg, "ollama_embed_model", "nomic-embed-text"),
|
||||
embed_timeout_sec=float(getattr(cfg, "llm_embed_timeout_sec", 10.0)),
|
||||
context_hint=context_hint,
|
||||
)
|
||||
routed_tools = _route_tools(cfg, redacted, _builtin_tools, mcp_tools, recent_messages)
|
||||
# Don't cache the router's "fall open to all tools" fallback. That
|
||||
# path fires when the LLM router times out, returns empty, or emits
|
||||
# a response no token of which matches a known tool name — i.e. the
|
||||
@@ -1217,6 +1317,9 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
"deleteMeal", "localFiles", "controlBrowser", "browseAndPlay", "screenshot",
|
||||
}
|
||||
_router_wants_data = any(t in routed_tools for t in _DATA_TOOLS)
|
||||
# Intent judge (v2): the planner only runs on turns that need a tool (the chat tool aside,
|
||||
# which is offered on every turn below); a turn without one goes straight to its answer.
|
||||
_turn_needs_tools = any(t not in ("chatChannels", "stop", "toolSearchTool") for t in routed_tools)
|
||||
|
||||
# In screen-share mode, always offer setBroadcast. "Turn the broadcast
|
||||
# on/off" is language-agnostic intent the embedding/keyword router won't
|
||||
@@ -1261,13 +1364,16 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
|
||||
action_plan: list[str] = []
|
||||
try:
|
||||
action_plan = plan_query(
|
||||
cfg=cfg,
|
||||
query=redacted,
|
||||
dialogue_context=_dialogue_ctx,
|
||||
tools=_planner_tool_catalog,
|
||||
live_context=_live_time_location_string(cfg),
|
||||
)
|
||||
# Intent judge (v2): the router decides whether the turn needs tools; only then does the
|
||||
# planner order the steps (planner.spec.md, "Intent judge (v2)").
|
||||
if not intent_judge or _turn_needs_tools:
|
||||
action_plan = plan_query(
|
||||
cfg=cfg,
|
||||
query=redacted,
|
||||
dialogue_context=_dialogue_ctx,
|
||||
tools=_planner_tool_catalog,
|
||||
live_context=_live_time_location_string(cfg),
|
||||
)
|
||||
except Exception as _plan_exc: # pragma: no cover — defensive
|
||||
debug_log(f"planner step failed (non-fatal): {_plan_exc}", "planning")
|
||||
action_plan = []
|
||||
@@ -1717,7 +1823,8 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
# _resolve_output_language). Building the persona from the raw env var while
|
||||
# the directive used the config value made the two contradict each other.
|
||||
_output_language = _resolve_output_language()
|
||||
_persona_prompt = build_system_prompt(_assistant_name, _output_language, default_language)
|
||||
_persona_prompt = (build_voice_persona if compact_prompt else build_system_prompt)(
|
||||
_assistant_name, _output_language, default_language)
|
||||
# File-based operator instructions: every *.md in AGENTS_DIR (default
|
||||
# /app/agents, bind-mounted from ./agents). Read once per turn so edits in
|
||||
# the folder apply on the next reply without a restart; fail-open to "".
|
||||
@@ -1726,8 +1833,18 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
def _build_initial_system_message() -> str:
|
||||
guidance = [_persona_prompt.strip()]
|
||||
|
||||
# Add model-size-appropriate prompt components
|
||||
guidance.extend(prompts.to_list())
|
||||
if compact_prompt:
|
||||
# The persona already covers voice style, inference and tool use; the tool guidance
|
||||
# (named entities, auto-derived and self-contained arguments) matters only when a
|
||||
# data tool is on offer. The sentinels and the always-offered chat tool are not.
|
||||
guidance.append(prompts.asr_note)
|
||||
if any(t not in ("stop", "toolSearchTool", "chatChannels") for t in allowed_tools):
|
||||
guidance.extend(c for c in (prompts.tool_guidance, prompts.tool_constraints) if c)
|
||||
else:
|
||||
guidance.append(COMPACT_ACTION_RULE)
|
||||
else:
|
||||
# Add model-size-appropriate prompt components
|
||||
guidance.extend(prompts.to_list())
|
||||
|
||||
# Reply-language policy: an explicit OUTPUT_LANGUAGE lock wins, else
|
||||
# Piper/Chatterbox TTS forces English (English-only voices), else the
|
||||
|
||||
@@ -204,6 +204,30 @@ The engine consumes the plan in two phases.
|
||||
the synthetic assistant `tool_calls` message, and no progress nudge is
|
||||
appended.
|
||||
|
||||
### Intent judge (v2)
|
||||
|
||||
The v2 brain (`src/voice_worker/voice_worker.spec.md`, "History and brain") judges each turn once
|
||||
before the answer: `run_reply_engine(intent_judge=True)` runs the tool router as usual (the harness's
|
||||
judgement of whether the turn needs a tool and which ones) and runs the planner **only when the
|
||||
router picked a tool** other than the sentinels (`stop`, `toolSearchTool`) and the always-offered
|
||||
`chatChannels`. A turn without a tool (greetings, small talk, facts the model knows) goes straight
|
||||
to its answer with one LLM call before it; a tool turn keeps the planner's ordered steps, which
|
||||
multi-step asks need ("who directed Possessor, and what else did they make").
|
||||
|
||||
Measured on Strata (Qwen3.6, 2026-11-12): the router took 0.42 s and the planner 0.49-0.94 s, one
|
||||
after the other, before every answer; each call costs at least one full prompt read. Tried first
|
||||
and dropped: the planner alone over the whole catalogue (one call) planned "reply with the
|
||||
weather" without a tool step and lost the weather tool (20 eval failures against 4), and the router
|
||||
alone without a planner failed the chained Possessor research 3 of 3 times. With this rule the
|
||||
whole eval suite passes as with v1's flow (`EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1`).
|
||||
|
||||
`prepare_intent(cfg, text, dialogue_memory, chat_available)` runs the same router pick ahead of
|
||||
time, with the inputs the reply would give it (the catalogue with `chatChannels` when a chat backend
|
||||
will be there, the context hint from the asker's dialogue memory); None when the router gave up (its
|
||||
fall-open to the whole catalogue is not a pick). The pick is passed back as
|
||||
`run_reply_engine(prepared_tools=...)` and the router is not run again. The brain calls it on a
|
||||
partial transcript while the user is still talking.
|
||||
|
||||
### resolve_next_tool_call
|
||||
|
||||
- **Fast path**: if the step text is fully concrete (tool name in the
|
||||
|
||||
@@ -157,6 +157,57 @@ _SYSTEM_PROMPT_TEMPLATE: str = (
|
||||
)
|
||||
|
||||
|
||||
# The v2 voice persona (voice_worker.spec.md, "History and brain"): the same butler in a tenth of
|
||||
# the words. v1's persona spends most of its length on the single-user long-term memory (profile,
|
||||
# diary, graph) and on the on-screen browser, neither of which the v2 brain has; Strata reads every
|
||||
# prompt from the start, so each dropped token is latency saved on every reply. The user asked for
|
||||
# answers as short as possible (2026-11-12): the aside is optional and only when nearly free.
|
||||
_VOICE_PERSONA_TEMPLATE: str = (
|
||||
"You are {name}, the voice assistant in a group voice chat, with the manner of a composed, "
|
||||
"polite butler and a dry sense of humour. "
|
||||
"Keep every reply as short as it can be while still answering well: exactly one short "
|
||||
"sentence that gives the answer first, with no second sentence even for a remark or advice "
|
||||
"(join them into the one sentence or drop one). No preamble, no repeating the question, no "
|
||||
"follow-up offers, no lists. Meet small talk in kind, warmly and briefly. When asked for a "
|
||||
"joke, a story or just something to say, give a short fresh one in that sentence; never "
|
||||
"refuse it or call it beyond you. "
|
||||
"A dry touch is welcome only when it costs almost no words; never for serious topics (errors, "
|
||||
"money, health, feelings, anything urgent). "
|
||||
"Never be mean, condescending or sycophantic ('great question'), never call the user 'sir' or "
|
||||
"'madam', and never answer with a bare greeting: engage with what was actually said. "
|
||||
"Your reply is spoken aloud by text-to-speech: plain words only, no markdown, emoji or code. "
|
||||
"{tool_rules} "
|
||||
"The [Context: ...] line at the top holds the real current time and location: answer time and "
|
||||
"date questions from it, phrased naturally in the user's language; never say you lack a clock. "
|
||||
"When 'Information shared in previous conversations' is present it is your memory of earlier "
|
||||
"conversations: what was said there is not a live fact and needs no tool, so answer questions "
|
||||
"about it from that section directly, and never say you cannot check or remember it."
|
||||
)
|
||||
|
||||
|
||||
def build_voice_persona(
|
||||
assistant_name: str = "Jarvis", output_language: Optional[str] = None,
|
||||
default_language: Optional[str] = None,
|
||||
) -> str:
|
||||
"""The short v2 voice persona; language handling as in ``build_system_prompt``."""
|
||||
name = (assistant_name or "Jarvis").strip() or "Jarvis"
|
||||
prompt = _VOICE_PERSONA_TEMPLATE.format(name=name, tool_rules=TOOL_HONESTY_RULES)
|
||||
lang = (output_language or "").strip()
|
||||
if lang:
|
||||
prompt = prompt.replace("in the user's language", f"in {lang}")
|
||||
elif (default_language or "").strip():
|
||||
prompt = prompt.replace("in the user's language", "in the reply language")
|
||||
return prompt
|
||||
|
||||
|
||||
# Compact prompts, turns where no data tool was routed: what the tool guidance says about acting
|
||||
# instead of refusing, in one line (the full guidance is only for turns with data tools).
|
||||
COMPACT_ACTION_RULE: str = (
|
||||
"If the user asks for an action or for live information that your tools here do not cover, "
|
||||
"call toolSearchTool with a short description of it before saying you cannot."
|
||||
)
|
||||
|
||||
|
||||
def tool_honesty_reminder(tool_names) -> str:
|
||||
"""End-of-prompt restatement of the tool honesty rules for this turn.
|
||||
|
||||
|
||||
@@ -14,8 +14,10 @@ except ImportError: # pragma: no cover - Python < 3.9
|
||||
ZoneInfoNotFoundError = Exception # type: ignore[assignment,misc]
|
||||
|
||||
|
||||
_UTC_FORMAT = "%A, %B %d, %Y at %H:%M UTC"
|
||||
_LOCAL_FORMAT = "%A, %B %d, %Y at %H:%M %Z"
|
||||
# The 12-hour form beside the 24-hour one: field run 2026-11-12 (Qwen3.6, Korean) read 10:37 as
|
||||
# 오후 (pm) 4 times of 4 with the short v2 prompt, and as 오전 (am) every time with it given.
|
||||
_UTC_FORMAT = "%A, %B %d, %Y at %H:%M (%I:%M %p) UTC"
|
||||
_LOCAL_FORMAT = "%A, %B %d, %Y at %H:%M (%I:%M %p) %Z"
|
||||
|
||||
|
||||
def format_time_context(
|
||||
|
||||
@@ -4,7 +4,9 @@ interface, so the worker logic is tested with a fake (voice_worker.spec.md, "His
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import difflib
|
||||
import sys
|
||||
import unicodedata
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
@@ -28,6 +30,11 @@ class Brain(Protocol):
|
||||
"""Blocking: candidate ids (``{"id", "user", "text"}``) most important first; None on failure.
|
||||
Runs on its own thread, also while a reply runs."""
|
||||
|
||||
def prepare(self, user: str, text: str, chat: bool) -> None:
|
||||
"""Blocking: get ready to answer ``text`` (a partial transcript of a call still being said),
|
||||
so the reply to about the same text starts further along. ``chat``: chat channels are
|
||||
registered. Runs on its own thread, also while a reply runs. Optional."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class MemorySettings:
|
||||
@@ -110,6 +117,32 @@ class Muted:
|
||||
cls._depth -= 1
|
||||
|
||||
|
||||
# A prepared intent is used for a reply whose text is at least this close to the text it was
|
||||
# prepared for (letters and digits only): the last partial often lacks a final syllable or two.
|
||||
PREPARED_SIMILARITY = 0.85
|
||||
# ... and only this soon after it was prepared, and at most this long is waited for one in flight.
|
||||
PREPARED_MAX_AGE_S = 30.0
|
||||
PREPARED_WAIT_S = 5.0
|
||||
|
||||
|
||||
def _letters(text: str) -> str:
|
||||
return "".join(c for c in unicodedata.normalize("NFKD", text.lower()) if c.isalnum())
|
||||
|
||||
|
||||
def similar(a: str, b: str) -> bool:
|
||||
x, y = _letters(a), _letters(b)
|
||||
return bool(x and y) and difflib.SequenceMatcher(None, x, y).ratio() >= PREPARED_SIMILARITY
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Prepared:
|
||||
text: str
|
||||
turns: int # the asker's dialogue memory size it was made with
|
||||
at: float
|
||||
done: threading.Event = dataclasses.field(default_factory=threading.Event)
|
||||
tools: Optional[list] = None # the intent judge's pick; None: none made
|
||||
|
||||
|
||||
class JarvisBrain:
|
||||
"""The existing jarvis reply engine with the user's history context.
|
||||
|
||||
@@ -125,13 +158,16 @@ class JarvisBrain:
|
||||
# v1's tts_engine names the v1 desktop voice (English-only Piper by default), which makes the
|
||||
# engine lock replies to English. v2 speaks through the worker's TTS and chat answers are
|
||||
# not spoken; the brain answers in ``reply_language`` (an OUTPUT_LANGUAGE lock still wins).
|
||||
self.cfg = dataclasses.replace(load_settings(), tts_engine="melo")
|
||||
# stream_browser: v1's screen-share browser (controlBrowser, browseAndPlay, setBroadcast and
|
||||
# browser-first web search) needs v1's Node bot; v2 has no on-screen browser until M6.
|
||||
self.cfg = dataclasses.replace(load_settings(), tts_engine="melo", stream_browser=False)
|
||||
self.db = Database(self.cfg.db_path, self.cfg.sqlite_vss_path)
|
||||
# ``language``: the ISO code tools get (e.g. the Wikipedia host); ``reply_language``: replies
|
||||
# come in it unless the asker explicitly asks for another language (None: the asker's).
|
||||
self.language = language
|
||||
self.reply_language = reply_language
|
||||
self._memories: dict = {}
|
||||
self._prepared: dict[str, _Prepared] = {}
|
||||
self._lock = threading.Lock()
|
||||
debug_log(f"brain ready: llm={self.cfg.ollama_base_url} model={self.cfg.ollama_chat_model}", "voice_worker")
|
||||
|
||||
@@ -145,11 +181,41 @@ class JarvisBrain:
|
||||
self._memories[user] = mem
|
||||
return mem
|
||||
|
||||
@staticmethod
|
||||
def _turns(mem) -> int:
|
||||
return len(mem.get_recent_messages()) if mem.has_recent_messages() else 0
|
||||
|
||||
def prepare(self, user: str, text: str, chat: bool) -> None:
|
||||
from jarvis.reply.engine import prepare_intent
|
||||
|
||||
mem = self._memory(user)
|
||||
p = _Prepared(text=text, turns=self._turns(mem), at=time.monotonic())
|
||||
with self._lock:
|
||||
self._prepared[user] = p
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
p.tools = Muted.run(prepare_intent, self.cfg, text, mem, chat)
|
||||
finally:
|
||||
p.done.set()
|
||||
debug_log(f"brain prepare user={user} picked={p.tools is not None} "
|
||||
f"ms={(time.perf_counter() - t0) * 1000:.0f}", "voice_worker")
|
||||
|
||||
def _prepared_tools(self, user: str, text: str) -> Optional[list]:
|
||||
"""The pick prepared for about this text, waiting for one still being made; None if none fits."""
|
||||
with self._lock:
|
||||
p = self._prepared.pop(user, None)
|
||||
if p is None or not similar(p.text, text) or time.monotonic() - p.at > PREPARED_MAX_AGE_S:
|
||||
return None
|
||||
if not p.done.wait(PREPARED_WAIT_S) or p.turns != self._turns(self._memory(user)):
|
||||
return None
|
||||
return p.tools
|
||||
|
||||
def reply(self, user: str, text: str, history: HistoryContext, chat=None) -> Optional[str]:
|
||||
from jarvis.reply import turn_state
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
|
||||
t0 = time.perf_counter()
|
||||
prepared = self._prepared_tools(user, text)
|
||||
# legacy_memory=False: v1's profile, diary and graph belong to one user; several people
|
||||
# share this assistant, so only the asker's scoped history may reach the prompt.
|
||||
def turn():
|
||||
@@ -160,13 +226,14 @@ class JarvisBrain:
|
||||
try:
|
||||
return run_reply_engine(self.db, self.cfg, None, text, self._memory(user),
|
||||
language=self.language, history=history, legacy_memory=False,
|
||||
default_language=self.reply_language)
|
||||
default_language=self.reply_language, compact_prompt=True,
|
||||
intent_judge=True, prepared_tools=prepared)
|
||||
finally:
|
||||
turn_state.reset()
|
||||
|
||||
answer = Muted.run(turn)
|
||||
debug_log(f"brain reply user={user} ok={bool(answer)} ms={(time.perf_counter() - t0) * 1000:.0f}",
|
||||
"voice_worker")
|
||||
debug_log(f"brain reply user={user} ok={bool(answer)} prepared={prepared is not None} "
|
||||
f"ms={(time.perf_counter() - t0) * 1000:.0f}", "voice_worker")
|
||||
return answer
|
||||
|
||||
def addressed(self, user: str, text: str, names: list[str]) -> Optional[bool]:
|
||||
|
||||
@@ -176,6 +176,9 @@ class Session:
|
||||
# Spoken replies whose speech was cancelled while the brain was still answering.
|
||||
self.cancelled_replies: set[str] = set()
|
||||
self.thinking_replies: set[str] = set()
|
||||
# user -> the newest call text waiting for that user's running prepare to finish (or None).
|
||||
self.preparing: dict[str, str | None] = {}
|
||||
self.last_prepared: dict[str, str] = {} # user -> the text of their last prepare
|
||||
# Answers sent but not yet known to have reached the person: id -> (user, turn, timer).
|
||||
self.deliveries: dict[str, tuple[str, int, asyncio.TimerHandle | None]] = {}
|
||||
self._handed_over: dict[str, bool] = {} # replies whose end is counted by _delivered
|
||||
@@ -316,7 +319,8 @@ class Session:
|
||||
"chat_delete": self._on_chat_delete, "reply": self._on_reply,
|
||||
"addressed": self._on_addressed, "rank": self._on_rank,
|
||||
"chat_channels": self._on_chat_channels, "chat_action_result": self._on_chat_action_result,
|
||||
"memory_config": self._on_memory_config, "reply_delivered": self._on_reply_delivered}.get(kind)
|
||||
"memory_config": self._on_memory_config, "reply_delivered": self._on_reply_delivered,
|
||||
"prepare": self._on_prepare}.get(kind)
|
||||
if handler is None:
|
||||
self.error("bad_message", f"unknown type: {kind!r}")
|
||||
return
|
||||
@@ -507,11 +511,47 @@ class Session:
|
||||
return
|
||||
if speak:
|
||||
self.thinking_replies.add(rid)
|
||||
if user in self.preparing:
|
||||
self.preparing[user] = None # the call was answered: no newer text to prepare
|
||||
self.last_prepared.pop(user, None)
|
||||
waiting = self.worker.replies_unanswered
|
||||
early = waiting.get(user, 0) > 0 # sent before their previous answer: not a reply to it
|
||||
waiting[user] = waiting.get(user, 0) + 1
|
||||
asyncio.get_running_loop().create_task(self._reply(rid, user, text, via, speak, early))
|
||||
|
||||
def _on_prepare(self, msg: dict) -> None:
|
||||
"""A call still being said: the brain gets ready for it. One prepare per user runs at a time;
|
||||
while it runs only the newest text waits, and one about the same as the last one prepared
|
||||
(or being prepared) is dropped. Nothing is sent back."""
|
||||
from .brain import similar
|
||||
got = self._strings(msg, "user", "text")
|
||||
if got is None:
|
||||
return
|
||||
user, text = got
|
||||
if self.worker.brain is None or not hasattr(self.worker.brain, "prepare"):
|
||||
return
|
||||
if user in self.last_prepared and similar(self.last_prepared[user], text):
|
||||
return
|
||||
if user in self.preparing:
|
||||
self.preparing[user] = text
|
||||
return
|
||||
self.preparing[user] = None
|
||||
asyncio.get_running_loop().create_task(self._prepare(user, text))
|
||||
|
||||
async def _prepare(self, user: str, text: str | None) -> None:
|
||||
from .brain import similar
|
||||
try:
|
||||
while text is not None and not self.closed:
|
||||
self.last_prepared[user] = text
|
||||
try:
|
||||
await asyncio.to_thread(self.worker.brain.prepare, user, text, bool(self.chat_channels))
|
||||
except Exception as e: # the brain must never take the session down
|
||||
debug_log(f"brain prepare failed: {type(e).__name__}", "voice_worker")
|
||||
newer, self.preparing[user] = self.preparing[user], None
|
||||
text = newer if newer is not None and not similar(newer, self.last_prepared.get(user, "")) else None
|
||||
finally:
|
||||
self.preparing.pop(user, None)
|
||||
|
||||
def _on_addressed(self, msg: dict) -> None:
|
||||
got = self._strings(msg, "id", "user", "text")
|
||||
if got is None:
|
||||
|
||||
@@ -201,6 +201,7 @@ which carry Discord's own time.
|
||||
-> {"type": "reply", "id": "r1", "user": "<id>", "text": "...", "via": "voice", "speak": true}
|
||||
<- {"type": "reply_text", "id": "r1", "text": "...", "failed": false}
|
||||
-> {"type": "reply_delivered", "id": "c1", "ok": true} // a chat answer was posted (ok: false: it was not)
|
||||
-> {"type": "prepare", "user": "<id>", "text": "..."} // a call still being said; nothing comes back
|
||||
-> {"type": "addressed", "id": "q1", "user": "<id>", "text": "..."}
|
||||
<- {"type": "addressed_result", "id": "q1", "addressed": true, "failed": false}
|
||||
-> {"type": "rank", "id": "k1", "candidates": [{"id": "7", "user": "<id>", "text": "..."}]}
|
||||
@@ -241,6 +242,23 @@ which carry Discord's own time.
|
||||
default) does not lock replies to English, because v2 speaks through the worker's TTS. An
|
||||
`output_language` lock in the jarvis settings still wins. Eval: `evals/test_reply_language_default.py`.
|
||||
STT (`language` `"ko"`) and TTS (`--melo-language KR`) are Korean by default too.
|
||||
Two reply options keep the time to the first word short (Strata reads every prompt from the
|
||||
start, about 0.36 s per 2,048 tokens, and each LLM call before the answer adds its whole time):
|
||||
the short voice persona (`compact_prompt=True`: the same butler in a tenth of v1's words, without
|
||||
v1's single-user memory and on-screen browser rules, answers as short as possible, the full tool
|
||||
guidance only on turns that offer a data tool), and the intent judge (`intent_judge=True`: the tool
|
||||
router judges whether the turn needs tools and which, and the planner runs only when it does;
|
||||
`planner.spec.md`, "Intent judge (v2)"). v1's screen-share browser (`stream_browser`) is off: it
|
||||
needs v1's Node bot (M6 brings the v2 one), so its tools are not offered.
|
||||
- `prepare` tells the brain that `user` is saying a call whose partial transcript is `text` (the app
|
||||
sends it while the speaker is still talking, `app/src/conversation.spec.md`). The brain runs the
|
||||
intent judge's tool pick for it on its own thread, with the asker's dialogue memory and the tool
|
||||
catalogue the reply will have. A `reply` from that user whose text is about the same (letters and
|
||||
digits, 85% alike), within 30 s and with the dialogue memory unchanged, uses that pick instead of
|
||||
making its own, waiting up to 5 s for one still being made. Each prepared judgement is used at most once.
|
||||
One `prepare` per user runs at a time: while it runs only the newest text waits, a text about the
|
||||
same as the one last prepared is dropped, and a `reply` from that user drops the waiting text.
|
||||
Nothing is sent back; without a brain, or with one that cannot prepare, it is ignored.
|
||||
The asker is set in `jarvis.reply.turn_state` for the turn, so tools that keep personal data
|
||||
(meals) store and read it under that person only.
|
||||
- `addressed` and `rank` are the conversation's questions (`app/src/conversation.spec.md`), answered
|
||||
|
||||
Reference in New Issue
Block a user