Alone with Jarvis, whatever the speaker says is answered without asking the judge. With others present the judge is told how many people are in the channel and how many may call Jarvis: with one other person most of what the speaker says counts as said to Jarvis (unclear is yes); with more, or not known, the question is the group one it was before. Live 2026-11-16: alone in the channel, questions without the wake word got no answer because the judge assumed a group. Eval (qwen3:8b, pair cases): 3/7 before, 6/7 after; group cases unchanged. Also: the protocol test fixture's brain takes the trace argument (the test failed since traces). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
348 lines
16 KiB
Python
348 lines
16 KiB
Python
"""The brain the voice worker hosts: memory settings and the jarvis reply engine behind a small
|
|
interface, so the worker logic is tested with a fake (voice_worker.spec.md, "History and brain")."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
import difflib
|
|
import sys
|
|
import unicodedata
|
|
import threading
|
|
import time
|
|
from dataclasses import dataclass
|
|
from typing import Optional, Protocol
|
|
|
|
from .log import debug_log, notice
|
|
from jarvis.memory.history import HistoryContext, Scope
|
|
|
|
|
|
class Brain(Protocol):
|
|
def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None,
|
|
task: bool = False, notes: Optional[list] = None, notes_backend=None,
|
|
screen=None, trace: Optional[list] = None) -> Optional[str]:
|
|
"""Blocking; called on a worker thread, one call per person at a time (a voice and a chat
|
|
reply for different people may overlap). ``chat``: the chat backend for
|
|
the chatChannels tool (voice_worker/chat_bridge.py), or None without registered channels.
|
|
``schedule``: the schedule backend for the schedule tool (voice_worker/app_bridge.py).
|
|
``task``: a scheduled task runs, with the read-only tools only (``TASK_TOOLS``).
|
|
``notes``: the asker's standing instructions, put in the prompt; ``notes_backend``: the
|
|
backend for the personalNotes tool (personal_notes.spec.md); ``screen``: the backend for the
|
|
screenView tool (screen_view.spec.md); ``trace``: a list the engine adds the steps of
|
|
this answer to (history.spec.md, "Traces")."""
|
|
|
|
def addressed(self, user: str, text: str, names: list[str], people: Optional[int] = None,
|
|
callers: Optional[int] = None) -> Optional[bool]:
|
|
"""Blocking: was ``text`` said to the assistant (called ``names``)? ``people``: how many are in
|
|
the voice channel, the speaker included; ``callers``: how many of them may call it (None:
|
|
not known). None when it could not tell. Runs on its own thread, also while a reply runs."""
|
|
|
|
def rank(self, candidates: list[dict]) -> Optional[list[str]]:
|
|
"""Blocking: candidate ids (``{"id", "user", "text"}``) most important first; None on failure.
|
|
Runs on its own thread, also while a reply runs."""
|
|
|
|
def prepare(self, user: str, text: str, chat: bool) -> None:
|
|
"""Blocking: get ready to answer ``text`` (a partial transcript of a call still being said),
|
|
so the reply to about the same text starts further along. ``chat``: chat channels are
|
|
registered. Runs on its own thread, also while a reply runs. Optional."""
|
|
|
|
|
|
@dataclass
|
|
class MemorySettings:
|
|
chat: bool = True
|
|
others: bool = True
|
|
off: bool = False
|
|
since_days: Optional[float] = None
|
|
recent_minutes: float = 30
|
|
recent_records: int = 20
|
|
matches: int = 5
|
|
budget_tokens: float = 4000
|
|
|
|
_BOOLS = ("chat", "others", "off")
|
|
_NUMBERS = {"recent_minutes": (0, 24 * 60), "recent_records": (0, 200), "matches": (0, 20),
|
|
"budget_tokens": (0, 100_000)}
|
|
|
|
def update(self, changes: dict) -> None:
|
|
"""Apply ``changes`` atomically; raises ValueError naming the first bad field."""
|
|
new = dict(self.__dict__)
|
|
for key, value in changes.items():
|
|
if key in self._BOOLS:
|
|
if not isinstance(value, bool):
|
|
raise ValueError(f"{key} must be true or false")
|
|
elif key == "since_days":
|
|
if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float))
|
|
or value <= 0):
|
|
raise ValueError("since_days must be a positive number or null")
|
|
elif key in self._NUMBERS:
|
|
lo, hi = self._NUMBERS[key]
|
|
if isinstance(value, bool) or not isinstance(value, (int, float)) or not lo <= value <= hi:
|
|
raise ValueError(f"{key} must be a number from {lo} to {hi}")
|
|
if key in ("recent_records", "matches"):
|
|
value = int(value)
|
|
else:
|
|
raise ValueError(f"unknown memory setting: {key}")
|
|
new[key] = value
|
|
self.__dict__.update(new)
|
|
|
|
def scope(self, now: float) -> Scope:
|
|
since = now - self.since_days * 86400 if self.since_days else None
|
|
return Scope(chat=self.chat, others=self.others, since=since, off=self.off)
|
|
|
|
|
|
class Muted:
|
|
"""Stand-in for sys.stdout/sys.stderr that drops every write while a brain reply runs.
|
|
|
|
The v1 reply engine and its tools print the question, the answer, tool arguments, URLs and
|
|
memory snippets (``print`` and ``debug_log``, also from helper threads such as the web fetch
|
|
pool); the worker's logs must never carry such text. Replies may overlap (a voice and a chat
|
|
lane); output stays muted while any of them runs. The worker's own log (``voice_worker.log``)
|
|
writes to the original stderr and is not muted.
|
|
"""
|
|
|
|
_lock = threading.Lock()
|
|
_depth = 0
|
|
|
|
def __init__(self, target):
|
|
self._target = target
|
|
|
|
def write(self, data):
|
|
if Muted._depth:
|
|
return len(data)
|
|
return self._target.write(data)
|
|
|
|
def __getattr__(self, name):
|
|
return getattr(self._target, name)
|
|
|
|
@classmethod
|
|
def run(cls, fn, *args, **kwargs):
|
|
with cls._lock:
|
|
for name in ("stdout", "stderr"):
|
|
stream = getattr(sys, name)
|
|
if not isinstance(stream, cls):
|
|
setattr(sys, name, cls(stream))
|
|
cls._depth += 1
|
|
try:
|
|
return fn(*args, **kwargs)
|
|
finally:
|
|
with cls._lock:
|
|
cls._depth -= 1
|
|
|
|
|
|
# A prepared intent is used for a reply whose text is at least this close to the text it was
|
|
# prepared for (letters and digits only): the last partial often lacks a final syllable or two.
|
|
PREPARED_SIMILARITY = 0.85
|
|
# ... and only this soon after it was prepared, and at most this long is waited for one in flight.
|
|
PREPARED_MAX_AGE_S = 30.0
|
|
PREPARED_WAIT_S = 5.0
|
|
|
|
|
|
def _letters(text: str) -> str:
|
|
return "".join(c for c in unicodedata.normalize("NFKD", text.lower()) if c.isalnum())
|
|
|
|
|
|
def similar(a: str, b: str) -> bool:
|
|
x, y = _letters(a), _letters(b)
|
|
return bool(x and y) and difflib.SequenceMatcher(None, x, y).ratio() >= PREPARED_SIMILARITY
|
|
|
|
|
|
@dataclass
|
|
class _Prepared:
|
|
text: str
|
|
turns: int # the asker's dialogue memory size it was made with
|
|
at: float
|
|
done: threading.Event = dataclasses.field(default_factory=threading.Event)
|
|
tools: Optional[list] = None # the intent judge's pick; None: none made
|
|
|
|
|
|
# The app passes the MCP servers' environment values (API keys, kept in its encrypted secrets)
|
|
# here, as JSON ``{server: {NAME: value}}``, never in the config file.
|
|
MCP_ENV_VAR = "JARVIS_MCP_ENV"
|
|
# v1's host tools: the PC owner's home folder (read, write, delete) and screen. In v2 anyone in the
|
|
# voice channel or a registered chat channel asks, so they are never offered (M7 brings screen
|
|
# viewing with its own permissions).
|
|
V2_HOST_TOOLS = ("localFiles", "screenshot")
|
|
# The only tools a scheduled task may use: it runs with nobody there to ask, so it only looks
|
|
# things up (schedule.spec.md, "Tasks"). No MCP tools either: what they do is not known.
|
|
TASK_TOOLS = ("getWeather", "webSearch", "fetchWebPage", "fetchMeals", "collectedNews", "toolSearchTool")
|
|
|
|
|
|
def take_mcp_env() -> dict:
|
|
"""The app's MCP environment, removed from this process's environment so no child process
|
|
(MCP servers included) inherits another server's secrets."""
|
|
import json
|
|
import os
|
|
|
|
raw = os.environ.pop(MCP_ENV_VAR, None)
|
|
if not raw:
|
|
return {}
|
|
try:
|
|
env = json.loads(raw)
|
|
except ValueError:
|
|
notice("Warning: the MCP environment from the app is not JSON; ignored", "llm")
|
|
return {}
|
|
if not isinstance(env, dict):
|
|
return {}
|
|
return {str(k): {str(n): str(v) for n, v in vars_.items()} for k, vars_ in env.items()
|
|
if isinstance(vars_, dict)}
|
|
|
|
|
|
def _with_mcp_env(mcps: dict, env: dict) -> dict:
|
|
out = {}
|
|
for name, server in mcps.items():
|
|
server = dict(server)
|
|
if env.get(name):
|
|
server["env"] = {**(server.get("env") or {}), **env[name]}
|
|
out[name] = server
|
|
return out
|
|
|
|
|
|
def _connect_mcp(mcps: dict) -> None:
|
|
"""Discover the MCP servers' tools into the registry's cache (the reply engine offers them
|
|
from there). Runs on its own thread so a slow server does not hold up the voice."""
|
|
from jarvis.tools.registry import initialize_mcp_tools
|
|
|
|
t0 = time.perf_counter()
|
|
try:
|
|
tools, errors = initialize_mcp_tools(mcps, verbose=False)
|
|
except Exception as e: # noqa: BLE001 - the brain answers without MCP tools
|
|
notice(f"Warning: MCP servers could not be connected: {type(e).__name__}", "llm")
|
|
return
|
|
for name in mcps:
|
|
if name in errors:
|
|
notice(f"Warning: MCP server {name} failed: {errors[name]}", "llm")
|
|
else:
|
|
notice(f"MCP server {name}: {sum(1 for t in tools if t.startswith(f'{name}__'))} tool(s)", "llm")
|
|
debug_log(f"mcp ready: {len(tools)} tool(s) ms={(time.perf_counter() - t0) * 1000:.0f}", "llm")
|
|
|
|
|
|
class JarvisBrain:
|
|
"""The existing jarvis reply engine with the user's history context.
|
|
|
|
Uses the jarvis settings (``ollama_base_url`` is Strata's ``/v1`` in v2) and its database for
|
|
the long-term memory it already keeps. Each user has their own short-term dialogue memory so
|
|
tool carry-over follows the person who asked.
|
|
"""
|
|
|
|
def __init__(self, language: Optional[str] = "ko", reply_language: Optional[str] = "Korean",
|
|
mcp_env: Optional[dict] = None):
|
|
from jarvis.config import load_settings
|
|
from jarvis.memory.db import Database
|
|
|
|
# v1's tts_engine names the v1 desktop voice (English-only Piper by default), which makes the
|
|
# engine lock replies to English. v2 speaks through the worker's TTS and chat answers are
|
|
# not spoken; the brain answers in ``reply_language`` (an OUTPUT_LANGUAGE lock still wins).
|
|
# stream_browser: v1's screen-share browser (controlBrowser, browseAndPlay, setBroadcast and
|
|
# browser-first web search) needs v1's Node bot; v2 has no on-screen browser until M6.
|
|
# disabled_tools: the app's feature switches plus v1's host tools (V2_HOST_TOOLS).
|
|
base = load_settings()
|
|
self.cfg = dataclasses.replace(
|
|
base, tts_engine="melo", stream_browser=False,
|
|
disabled_tools=tuple(dict.fromkeys((*base.disabled_tools, *V2_HOST_TOOLS))),
|
|
mcps=_with_mcp_env(base.mcps, take_mcp_env() if mcp_env is None else mcp_env))
|
|
self.db = Database(self.cfg.db_path, self.cfg.sqlite_vss_path)
|
|
# ``language``: the ISO code tools get (e.g. the Wikipedia host); ``reply_language``: replies
|
|
# come in it unless the asker explicitly asks for another language (None: the asker's).
|
|
self.language = language
|
|
self.reply_language = reply_language
|
|
self._memories: dict = {}
|
|
self._prepared: dict[str, _Prepared] = {}
|
|
self._lock = threading.Lock()
|
|
debug_log(f"brain ready: llm={self.cfg.ollama_base_url} model={self.cfg.ollama_chat_model}", "llm")
|
|
if self.cfg.mcps:
|
|
threading.Thread(target=_connect_mcp, args=(self.cfg.mcps,), name="mcp-connect",
|
|
daemon=True).start()
|
|
|
|
def _memory(self, user: str):
|
|
from jarvis.memory.conversation import DialogueMemory
|
|
|
|
with self._lock:
|
|
mem = self._memories.get(user)
|
|
if mem is None:
|
|
mem = DialogueMemory(inactivity_timeout=self.cfg.dialogue_memory_timeout, max_interactions=20)
|
|
self._memories[user] = mem
|
|
return mem
|
|
|
|
@staticmethod
|
|
def _turns(mem) -> int:
|
|
return len(mem.get_recent_messages()) if mem.has_recent_messages() else 0
|
|
|
|
def prepare(self, user: str, text: str, chat: bool) -> None:
|
|
from jarvis.reply.engine import prepare_intent
|
|
|
|
mem = self._memory(user)
|
|
p = _Prepared(text=text, turns=self._turns(mem), at=time.monotonic())
|
|
with self._lock:
|
|
self._prepared[user] = p
|
|
t0 = time.perf_counter()
|
|
try:
|
|
p.tools = Muted.run(prepare_intent, self.cfg, text, mem, chat, schedule_available=True,
|
|
notes_available=True, screen_available=True)
|
|
finally:
|
|
p.done.set()
|
|
debug_log(f"brain prepare user={user} picked={p.tools is not None} "
|
|
f"ms={(time.perf_counter() - t0) * 1000:.0f}", "llm")
|
|
|
|
def _prepared_tools(self, user: str, text: str) -> Optional[list]:
|
|
"""The pick prepared for about this text, waiting for one still being made; None if none fits."""
|
|
with self._lock:
|
|
p = self._prepared.pop(user, None)
|
|
if p is None or not similar(p.text, text) or time.monotonic() - p.at > PREPARED_MAX_AGE_S:
|
|
return None
|
|
if not p.done.wait(PREPARED_WAIT_S) or p.turns != self._turns(self._memory(user)):
|
|
return None
|
|
return p.tools
|
|
|
|
def _task_cfg(self):
|
|
from jarvis.tools.registry import BUILTIN_TOOLS
|
|
|
|
off = (*self.cfg.disabled_tools, *(t for t in BUILTIN_TOOLS if t not in TASK_TOOLS))
|
|
return dataclasses.replace(self.cfg, disabled_tools=tuple(dict.fromkeys(off)), mcps={})
|
|
|
|
def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None,
|
|
task: bool = False, notes: Optional[list] = None, notes_backend=None,
|
|
screen=None, trace: Optional[list] = None) -> Optional[str]:
|
|
from jarvis.memory.conversation import DialogueMemory
|
|
from jarvis.reply import turn_state
|
|
from jarvis.reply.engine import run_reply_engine
|
|
|
|
t0 = time.perf_counter()
|
|
# A task has its own short-term memory: it is not a turn of the person's conversation.
|
|
prepared = None if task else self._prepared_tools(user, text)
|
|
cfg = self._task_cfg() if task else self.cfg
|
|
memory = DialogueMemory(inactivity_timeout=self.cfg.dialogue_memory_timeout, max_interactions=20) \
|
|
if task else self._memory(user)
|
|
# legacy_memory=False: v1's profile, diary and graph belong to one user; several people
|
|
# share this assistant, so only the asker's scoped history may reach the prompt.
|
|
def turn():
|
|
# Tools with personal data (meals) read the asker from the turn state.
|
|
turn_state.reset()
|
|
turn_state.set_user(user)
|
|
turn_state.set_chat(None if task else chat)
|
|
turn_state.set_schedule(None if task else schedule)
|
|
turn_state.set_notes(None if task else notes_backend)
|
|
turn_state.set_screen(None if task else screen)
|
|
turn_state.set_trace(trace)
|
|
try:
|
|
return run_reply_engine(self.db, cfg, None, text, memory,
|
|
language=self.language, history=history, legacy_memory=False,
|
|
default_language=self.reply_language, compact_prompt=True,
|
|
intent_judge=True, prepared_tools=prepared,
|
|
person_notes=notes or None)
|
|
finally:
|
|
turn_state.reset()
|
|
|
|
answer = Muted.run(turn)
|
|
debug_log(f"brain reply user={user} ok={bool(answer)} task={task} prepared={prepared is not None} "
|
|
f"ms={(time.perf_counter() - t0) * 1000:.0f}", "llm")
|
|
return answer
|
|
|
|
def addressed(self, user: str, text: str, names: list[str], people: Optional[int] = None,
|
|
callers: Optional[int] = None) -> Optional[bool]:
|
|
from .judges import is_addressed
|
|
return is_addressed(self.cfg.ollama_base_url, self.cfg.ollama_chat_model, names, text,
|
|
people=people, callers=callers)
|
|
|
|
def rank(self, candidates: list[dict]) -> Optional[list[str]]:
|
|
from .judges import rank_requests
|
|
order = rank_requests(self.cfg.ollama_base_url, self.cfg.ollama_chat_model, [c["text"] for c in candidates])
|
|
return None if order is None else [candidates[i]["id"] for i in order]
|