Files
javis_bot/src/voice_worker/brain.py
javis-bot 72518ca50e feat(app): addressed calls follow who is in the voice channel
Alone with Jarvis, whatever the speaker says is answered without asking the judge. With others
present the judge is told how many people are in the channel and how many may call Jarvis: with
one other person most of what the speaker says counts as said to Jarvis (unclear is yes); with
more, or not known, the question is the group one it was before. Live 2026-11-16: alone in the
channel, questions without the wake word got no answer because the judge assumed a group.
Eval (qwen3:8b, pair cases): 3/7 before, 6/7 after; group cases unchanged.

Also: the protocol test fixture's brain takes the trace argument (the test failed since traces).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 15:43:10 +09:00

348 lines
16 KiB
Python

"""The brain the voice worker hosts: memory settings and the jarvis reply engine behind a small
interface, so the worker logic is tested with a fake (voice_worker.spec.md, "History and brain")."""
from __future__ import annotations
import dataclasses
import difflib
import sys
import unicodedata
import threading
import time
from dataclasses import dataclass
from typing import Optional, Protocol
from .log import debug_log, notice
from jarvis.memory.history import HistoryContext, Scope
class Brain(Protocol):
def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None,
task: bool = False, notes: Optional[list] = None, notes_backend=None,
screen=None, trace: Optional[list] = None) -> Optional[str]:
"""Blocking; called on a worker thread, one call per person at a time (a voice and a chat
reply for different people may overlap). ``chat``: the chat backend for
the chatChannels tool (voice_worker/chat_bridge.py), or None without registered channels.
``schedule``: the schedule backend for the schedule tool (voice_worker/app_bridge.py).
``task``: a scheduled task runs, with the read-only tools only (``TASK_TOOLS``).
``notes``: the asker's standing instructions, put in the prompt; ``notes_backend``: the
backend for the personalNotes tool (personal_notes.spec.md); ``screen``: the backend for the
screenView tool (screen_view.spec.md); ``trace``: a list the engine adds the steps of
this answer to (history.spec.md, "Traces")."""
def addressed(self, user: str, text: str, names: list[str], people: Optional[int] = None,
callers: Optional[int] = None) -> Optional[bool]:
"""Blocking: was ``text`` said to the assistant (called ``names``)? ``people``: how many are in
the voice channel, the speaker included; ``callers``: how many of them may call it (None:
not known). None when it could not tell. Runs on its own thread, also while a reply runs."""
def rank(self, candidates: list[dict]) -> Optional[list[str]]:
"""Blocking: candidate ids (``{"id", "user", "text"}``) most important first; None on failure.
Runs on its own thread, also while a reply runs."""
def prepare(self, user: str, text: str, chat: bool) -> None:
"""Blocking: get ready to answer ``text`` (a partial transcript of a call still being said),
so the reply to about the same text starts further along. ``chat``: chat channels are
registered. Runs on its own thread, also while a reply runs. Optional."""
@dataclass
class MemorySettings:
chat: bool = True
others: bool = True
off: bool = False
since_days: Optional[float] = None
recent_minutes: float = 30
recent_records: int = 20
matches: int = 5
budget_tokens: float = 4000
_BOOLS = ("chat", "others", "off")
_NUMBERS = {"recent_minutes": (0, 24 * 60), "recent_records": (0, 200), "matches": (0, 20),
"budget_tokens": (0, 100_000)}
def update(self, changes: dict) -> None:
"""Apply ``changes`` atomically; raises ValueError naming the first bad field."""
new = dict(self.__dict__)
for key, value in changes.items():
if key in self._BOOLS:
if not isinstance(value, bool):
raise ValueError(f"{key} must be true or false")
elif key == "since_days":
if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float))
or value <= 0):
raise ValueError("since_days must be a positive number or null")
elif key in self._NUMBERS:
lo, hi = self._NUMBERS[key]
if isinstance(value, bool) or not isinstance(value, (int, float)) or not lo <= value <= hi:
raise ValueError(f"{key} must be a number from {lo} to {hi}")
if key in ("recent_records", "matches"):
value = int(value)
else:
raise ValueError(f"unknown memory setting: {key}")
new[key] = value
self.__dict__.update(new)
def scope(self, now: float) -> Scope:
since = now - self.since_days * 86400 if self.since_days else None
return Scope(chat=self.chat, others=self.others, since=since, off=self.off)
class Muted:
"""Stand-in for sys.stdout/sys.stderr that drops every write while a brain reply runs.
The v1 reply engine and its tools print the question, the answer, tool arguments, URLs and
memory snippets (``print`` and ``debug_log``, also from helper threads such as the web fetch
pool); the worker's logs must never carry such text. Replies may overlap (a voice and a chat
lane); output stays muted while any of them runs. The worker's own log (``voice_worker.log``)
writes to the original stderr and is not muted.
"""
_lock = threading.Lock()
_depth = 0
def __init__(self, target):
self._target = target
def write(self, data):
if Muted._depth:
return len(data)
return self._target.write(data)
def __getattr__(self, name):
return getattr(self._target, name)
@classmethod
def run(cls, fn, *args, **kwargs):
with cls._lock:
for name in ("stdout", "stderr"):
stream = getattr(sys, name)
if not isinstance(stream, cls):
setattr(sys, name, cls(stream))
cls._depth += 1
try:
return fn(*args, **kwargs)
finally:
with cls._lock:
cls._depth -= 1
# A prepared intent is used for a reply whose text is at least this close to the text it was
# prepared for (letters and digits only): the last partial often lacks a final syllable or two.
PREPARED_SIMILARITY = 0.85
# ... and only this soon after it was prepared, and at most this long is waited for one in flight.
PREPARED_MAX_AGE_S = 30.0
PREPARED_WAIT_S = 5.0
def _letters(text: str) -> str:
return "".join(c for c in unicodedata.normalize("NFKD", text.lower()) if c.isalnum())
def similar(a: str, b: str) -> bool:
x, y = _letters(a), _letters(b)
return bool(x and y) and difflib.SequenceMatcher(None, x, y).ratio() >= PREPARED_SIMILARITY
@dataclass
class _Prepared:
text: str
turns: int # the asker's dialogue memory size it was made with
at: float
done: threading.Event = dataclasses.field(default_factory=threading.Event)
tools: Optional[list] = None # the intent judge's pick; None: none made
# The app passes the MCP servers' environment values (API keys, kept in its encrypted secrets)
# here, as JSON ``{server: {NAME: value}}``, never in the config file.
MCP_ENV_VAR = "JARVIS_MCP_ENV"
# v1's host tools: the PC owner's home folder (read, write, delete) and screen. In v2 anyone in the
# voice channel or a registered chat channel asks, so they are never offered (M7 brings screen
# viewing with its own permissions).
V2_HOST_TOOLS = ("localFiles", "screenshot")
# The only tools a scheduled task may use: it runs with nobody there to ask, so it only looks
# things up (schedule.spec.md, "Tasks"). No MCP tools either: what they do is not known.
TASK_TOOLS = ("getWeather", "webSearch", "fetchWebPage", "fetchMeals", "collectedNews", "toolSearchTool")
def take_mcp_env() -> dict:
"""The app's MCP environment, removed from this process's environment so no child process
(MCP servers included) inherits another server's secrets."""
import json
import os
raw = os.environ.pop(MCP_ENV_VAR, None)
if not raw:
return {}
try:
env = json.loads(raw)
except ValueError:
notice("Warning: the MCP environment from the app is not JSON; ignored", "llm")
return {}
if not isinstance(env, dict):
return {}
return {str(k): {str(n): str(v) for n, v in vars_.items()} for k, vars_ in env.items()
if isinstance(vars_, dict)}
def _with_mcp_env(mcps: dict, env: dict) -> dict:
out = {}
for name, server in mcps.items():
server = dict(server)
if env.get(name):
server["env"] = {**(server.get("env") or {}), **env[name]}
out[name] = server
return out
def _connect_mcp(mcps: dict) -> None:
"""Discover the MCP servers' tools into the registry's cache (the reply engine offers them
from there). Runs on its own thread so a slow server does not hold up the voice."""
from jarvis.tools.registry import initialize_mcp_tools
t0 = time.perf_counter()
try:
tools, errors = initialize_mcp_tools(mcps, verbose=False)
except Exception as e: # noqa: BLE001 - the brain answers without MCP tools
notice(f"Warning: MCP servers could not be connected: {type(e).__name__}", "llm")
return
for name in mcps:
if name in errors:
notice(f"Warning: MCP server {name} failed: {errors[name]}", "llm")
else:
notice(f"MCP server {name}: {sum(1 for t in tools if t.startswith(f'{name}__'))} tool(s)", "llm")
debug_log(f"mcp ready: {len(tools)} tool(s) ms={(time.perf_counter() - t0) * 1000:.0f}", "llm")
class JarvisBrain:
"""The existing jarvis reply engine with the user's history context.
Uses the jarvis settings (``ollama_base_url`` is Strata's ``/v1`` in v2) and its database for
the long-term memory it already keeps. Each user has their own short-term dialogue memory so
tool carry-over follows the person who asked.
"""
def __init__(self, language: Optional[str] = "ko", reply_language: Optional[str] = "Korean",
mcp_env: Optional[dict] = None):
from jarvis.config import load_settings
from jarvis.memory.db import Database
# v1's tts_engine names the v1 desktop voice (English-only Piper by default), which makes the
# engine lock replies to English. v2 speaks through the worker's TTS and chat answers are
# not spoken; the brain answers in ``reply_language`` (an OUTPUT_LANGUAGE lock still wins).
# stream_browser: v1's screen-share browser (controlBrowser, browseAndPlay, setBroadcast and
# browser-first web search) needs v1's Node bot; v2 has no on-screen browser until M6.
# disabled_tools: the app's feature switches plus v1's host tools (V2_HOST_TOOLS).
base = load_settings()
self.cfg = dataclasses.replace(
base, tts_engine="melo", stream_browser=False,
disabled_tools=tuple(dict.fromkeys((*base.disabled_tools, *V2_HOST_TOOLS))),
mcps=_with_mcp_env(base.mcps, take_mcp_env() if mcp_env is None else mcp_env))
self.db = Database(self.cfg.db_path, self.cfg.sqlite_vss_path)
# ``language``: the ISO code tools get (e.g. the Wikipedia host); ``reply_language``: replies
# come in it unless the asker explicitly asks for another language (None: the asker's).
self.language = language
self.reply_language = reply_language
self._memories: dict = {}
self._prepared: dict[str, _Prepared] = {}
self._lock = threading.Lock()
debug_log(f"brain ready: llm={self.cfg.ollama_base_url} model={self.cfg.ollama_chat_model}", "llm")
if self.cfg.mcps:
threading.Thread(target=_connect_mcp, args=(self.cfg.mcps,), name="mcp-connect",
daemon=True).start()
def _memory(self, user: str):
from jarvis.memory.conversation import DialogueMemory
with self._lock:
mem = self._memories.get(user)
if mem is None:
mem = DialogueMemory(inactivity_timeout=self.cfg.dialogue_memory_timeout, max_interactions=20)
self._memories[user] = mem
return mem
@staticmethod
def _turns(mem) -> int:
return len(mem.get_recent_messages()) if mem.has_recent_messages() else 0
def prepare(self, user: str, text: str, chat: bool) -> None:
from jarvis.reply.engine import prepare_intent
mem = self._memory(user)
p = _Prepared(text=text, turns=self._turns(mem), at=time.monotonic())
with self._lock:
self._prepared[user] = p
t0 = time.perf_counter()
try:
p.tools = Muted.run(prepare_intent, self.cfg, text, mem, chat, schedule_available=True,
notes_available=True, screen_available=True)
finally:
p.done.set()
debug_log(f"brain prepare user={user} picked={p.tools is not None} "
f"ms={(time.perf_counter() - t0) * 1000:.0f}", "llm")
def _prepared_tools(self, user: str, text: str) -> Optional[list]:
"""The pick prepared for about this text, waiting for one still being made; None if none fits."""
with self._lock:
p = self._prepared.pop(user, None)
if p is None or not similar(p.text, text) or time.monotonic() - p.at > PREPARED_MAX_AGE_S:
return None
if not p.done.wait(PREPARED_WAIT_S) or p.turns != self._turns(self._memory(user)):
return None
return p.tools
def _task_cfg(self):
from jarvis.tools.registry import BUILTIN_TOOLS
off = (*self.cfg.disabled_tools, *(t for t in BUILTIN_TOOLS if t not in TASK_TOOLS))
return dataclasses.replace(self.cfg, disabled_tools=tuple(dict.fromkeys(off)), mcps={})
def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None,
task: bool = False, notes: Optional[list] = None, notes_backend=None,
screen=None, trace: Optional[list] = None) -> Optional[str]:
from jarvis.memory.conversation import DialogueMemory
from jarvis.reply import turn_state
from jarvis.reply.engine import run_reply_engine
t0 = time.perf_counter()
# A task has its own short-term memory: it is not a turn of the person's conversation.
prepared = None if task else self._prepared_tools(user, text)
cfg = self._task_cfg() if task else self.cfg
memory = DialogueMemory(inactivity_timeout=self.cfg.dialogue_memory_timeout, max_interactions=20) \
if task else self._memory(user)
# legacy_memory=False: v1's profile, diary and graph belong to one user; several people
# share this assistant, so only the asker's scoped history may reach the prompt.
def turn():
# Tools with personal data (meals) read the asker from the turn state.
turn_state.reset()
turn_state.set_user(user)
turn_state.set_chat(None if task else chat)
turn_state.set_schedule(None if task else schedule)
turn_state.set_notes(None if task else notes_backend)
turn_state.set_screen(None if task else screen)
turn_state.set_trace(trace)
try:
return run_reply_engine(self.db, cfg, None, text, memory,
language=self.language, history=history, legacy_memory=False,
default_language=self.reply_language, compact_prompt=True,
intent_judge=True, prepared_tools=prepared,
person_notes=notes or None)
finally:
turn_state.reset()
answer = Muted.run(turn)
debug_log(f"brain reply user={user} ok={bool(answer)} task={task} prepared={prepared is not None} "
f"ms={(time.perf_counter() - t0) * 1000:.0f}", "llm")
return answer
def addressed(self, user: str, text: str, names: list[str], people: Optional[int] = None,
callers: Optional[int] = None) -> Optional[bool]:
from .judges import is_addressed
return is_addressed(self.cfg.ollama_base_url, self.cfg.ollama_chat_model, names, text,
people=people, callers=callers)
def rank(self, candidates: list[dict]) -> Optional[list[str]]:
from .judges import rank_requests
order = rank_requests(self.cfg.ollama_base_url, self.cfg.ollama_chat_model, [c["text"] for c in candidates])
return None if order is None else [candidates[i]["id"] for i in order]