Files
javis_bot/src/voice_worker/brain.py
2026-11-16 03:00:10 +09:00

344 lines
16 KiB
Python

"""The brain the voice worker hosts: memory settings and the jarvis reply engine behind a small
interface, so the worker logic is tested with a fake (voice_worker.spec.md, "History and brain")."""
from __future__ import annotations
import dataclasses
import difflib
import sys
import unicodedata
import threading
import time
from dataclasses import dataclass
from typing import Optional, Protocol
from .log import debug_log, notice
from jarvis.memory.history import HistoryContext, Scope
class Brain(Protocol):
def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None,
task: bool = False, notes: Optional[list] = None, notes_backend=None,
screen=None, trace: Optional[list] = None) -> Optional[str]:
"""Blocking; called on a worker thread, one call per person at a time (a voice and a chat
reply for different people may overlap). ``chat``: the chat backend for
the chatChannels tool (voice_worker/chat_bridge.py), or None without registered channels.
``schedule``: the schedule backend for the schedule tool (voice_worker/app_bridge.py).
``task``: a scheduled task runs, with the read-only tools only (``TASK_TOOLS``).
``notes``: the asker's standing instructions, put in the prompt; ``notes_backend``: the
backend for the personalNotes tool (personal_notes.spec.md); ``screen``: the backend for the
screenView tool (screen_view.spec.md); ``trace``: a list the engine adds the steps of
this answer to (history.spec.md, "Traces")."""
def addressed(self, user: str, text: str, names: list[str]) -> Optional[bool]:
"""Blocking: was ``text`` said to the assistant (called ``names``)? None when it could not tell.
Runs on its own thread, also while a reply runs."""
def rank(self, candidates: list[dict]) -> Optional[list[str]]:
"""Blocking: candidate ids (``{"id", "user", "text"}``) most important first; None on failure.
Runs on its own thread, also while a reply runs."""
def prepare(self, user: str, text: str, chat: bool) -> None:
"""Blocking: get ready to answer ``text`` (a partial transcript of a call still being said),
so the reply to about the same text starts further along. ``chat``: chat channels are
registered. Runs on its own thread, also while a reply runs. Optional."""
@dataclass
class MemorySettings:
chat: bool = True
others: bool = True
off: bool = False
since_days: Optional[float] = None
recent_minutes: float = 30
recent_records: int = 20
matches: int = 5
budget_tokens: float = 4000
_BOOLS = ("chat", "others", "off")
_NUMBERS = {"recent_minutes": (0, 24 * 60), "recent_records": (0, 200), "matches": (0, 20),
"budget_tokens": (0, 100_000)}
def update(self, changes: dict) -> None:
"""Apply ``changes`` atomically; raises ValueError naming the first bad field."""
new = dict(self.__dict__)
for key, value in changes.items():
if key in self._BOOLS:
if not isinstance(value, bool):
raise ValueError(f"{key} must be true or false")
elif key == "since_days":
if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float))
or value <= 0):
raise ValueError("since_days must be a positive number or null")
elif key in self._NUMBERS:
lo, hi = self._NUMBERS[key]
if isinstance(value, bool) or not isinstance(value, (int, float)) or not lo <= value <= hi:
raise ValueError(f"{key} must be a number from {lo} to {hi}")
if key in ("recent_records", "matches"):
value = int(value)
else:
raise ValueError(f"unknown memory setting: {key}")
new[key] = value
self.__dict__.update(new)
def scope(self, now: float) -> Scope:
since = now - self.since_days * 86400 if self.since_days else None
return Scope(chat=self.chat, others=self.others, since=since, off=self.off)
class Muted:
"""Stand-in for sys.stdout/sys.stderr that drops every write while a brain reply runs.
The v1 reply engine and its tools print the question, the answer, tool arguments, URLs and
memory snippets (``print`` and ``debug_log``, also from helper threads such as the web fetch
pool); the worker's logs must never carry such text. Replies may overlap (a voice and a chat
lane); output stays muted while any of them runs. The worker's own log (``voice_worker.log``)
writes to the original stderr and is not muted.
"""
_lock = threading.Lock()
_depth = 0
def __init__(self, target):
self._target = target
def write(self, data):
if Muted._depth:
return len(data)
return self._target.write(data)
def __getattr__(self, name):
return getattr(self._target, name)
@classmethod
def run(cls, fn, *args, **kwargs):
with cls._lock:
for name in ("stdout", "stderr"):
stream = getattr(sys, name)
if not isinstance(stream, cls):
setattr(sys, name, cls(stream))
cls._depth += 1
try:
return fn(*args, **kwargs)
finally:
with cls._lock:
cls._depth -= 1
# A prepared intent is used for a reply whose text is at least this close to the text it was
# prepared for (letters and digits only): the last partial often lacks a final syllable or two.
PREPARED_SIMILARITY = 0.85
# ... and only this soon after it was prepared, and at most this long is waited for one in flight.
PREPARED_MAX_AGE_S = 30.0
PREPARED_WAIT_S = 5.0
def _letters(text: str) -> str:
return "".join(c for c in unicodedata.normalize("NFKD", text.lower()) if c.isalnum())
def similar(a: str, b: str) -> bool:
x, y = _letters(a), _letters(b)
return bool(x and y) and difflib.SequenceMatcher(None, x, y).ratio() >= PREPARED_SIMILARITY
@dataclass
class _Prepared:
text: str
turns: int # the asker's dialogue memory size it was made with
at: float
done: threading.Event = dataclasses.field(default_factory=threading.Event)
tools: Optional[list] = None # the intent judge's pick; None: none made
# The app passes the MCP servers' environment values (API keys, kept in its encrypted secrets)
# here, as JSON ``{server: {NAME: value}}``, never in the config file.
MCP_ENV_VAR = "JARVIS_MCP_ENV"
# v1's host tools: the PC owner's home folder (read, write, delete) and screen. In v2 anyone in the
# voice channel or a registered chat channel asks, so they are never offered (M7 brings screen
# viewing with its own permissions).
V2_HOST_TOOLS = ("localFiles", "screenshot")
# The only tools a scheduled task may use: it runs with nobody there to ask, so it only looks
# things up (schedule.spec.md, "Tasks"). No MCP tools either: what they do is not known.
TASK_TOOLS = ("getWeather", "webSearch", "fetchWebPage", "fetchMeals", "collectedNews", "toolSearchTool")
def take_mcp_env() -> dict:
"""The app's MCP environment, removed from this process's environment so no child process
(MCP servers included) inherits another server's secrets."""
import json
import os
raw = os.environ.pop(MCP_ENV_VAR, None)
if not raw:
return {}
try:
env = json.loads(raw)
except ValueError:
notice("Warning: the MCP environment from the app is not JSON; ignored", "llm")
return {}
if not isinstance(env, dict):
return {}
return {str(k): {str(n): str(v) for n, v in vars_.items()} for k, vars_ in env.items()
if isinstance(vars_, dict)}
def _with_mcp_env(mcps: dict, env: dict) -> dict:
out = {}
for name, server in mcps.items():
server = dict(server)
if env.get(name):
server["env"] = {**(server.get("env") or {}), **env[name]}
out[name] = server
return out
def _connect_mcp(mcps: dict) -> None:
"""Discover the MCP servers' tools into the registry's cache (the reply engine offers them
from there). Runs on its own thread so a slow server does not hold up the voice."""
from jarvis.tools.registry import initialize_mcp_tools
t0 = time.perf_counter()
try:
tools, errors = initialize_mcp_tools(mcps, verbose=False)
except Exception as e: # noqa: BLE001 - the brain answers without MCP tools
notice(f"Warning: MCP servers could not be connected: {type(e).__name__}", "llm")
return
for name in mcps:
if name in errors:
notice(f"Warning: MCP server {name} failed: {errors[name]}", "llm")
else:
notice(f"MCP server {name}: {sum(1 for t in tools if t.startswith(f'{name}__'))} tool(s)", "llm")
debug_log(f"mcp ready: {len(tools)} tool(s) ms={(time.perf_counter() - t0) * 1000:.0f}", "llm")
class JarvisBrain:
"""The existing jarvis reply engine with the user's history context.
Uses the jarvis settings (``ollama_base_url`` is Strata's ``/v1`` in v2) and its database for
the long-term memory it already keeps. Each user has their own short-term dialogue memory so
tool carry-over follows the person who asked.
"""
def __init__(self, language: Optional[str] = "ko", reply_language: Optional[str] = "Korean",
mcp_env: Optional[dict] = None):
from jarvis.config import load_settings
from jarvis.memory.db import Database
# v1's tts_engine names the v1 desktop voice (English-only Piper by default), which makes the
# engine lock replies to English. v2 speaks through the worker's TTS and chat answers are
# not spoken; the brain answers in ``reply_language`` (an OUTPUT_LANGUAGE lock still wins).
# stream_browser: v1's screen-share browser (controlBrowser, browseAndPlay, setBroadcast and
# browser-first web search) needs v1's Node bot; v2 has no on-screen browser until M6.
# disabled_tools: the app's feature switches plus v1's host tools (V2_HOST_TOOLS).
base = load_settings()
self.cfg = dataclasses.replace(
base, tts_engine="melo", stream_browser=False,
disabled_tools=tuple(dict.fromkeys((*base.disabled_tools, *V2_HOST_TOOLS))),
mcps=_with_mcp_env(base.mcps, take_mcp_env() if mcp_env is None else mcp_env))
self.db = Database(self.cfg.db_path, self.cfg.sqlite_vss_path)
# ``language``: the ISO code tools get (e.g. the Wikipedia host); ``reply_language``: replies
# come in it unless the asker explicitly asks for another language (None: the asker's).
self.language = language
self.reply_language = reply_language
self._memories: dict = {}
self._prepared: dict[str, _Prepared] = {}
self._lock = threading.Lock()
debug_log(f"brain ready: llm={self.cfg.ollama_base_url} model={self.cfg.ollama_chat_model}", "llm")
if self.cfg.mcps:
threading.Thread(target=_connect_mcp, args=(self.cfg.mcps,), name="mcp-connect",
daemon=True).start()
def _memory(self, user: str):
from jarvis.memory.conversation import DialogueMemory
with self._lock:
mem = self._memories.get(user)
if mem is None:
mem = DialogueMemory(inactivity_timeout=self.cfg.dialogue_memory_timeout, max_interactions=20)
self._memories[user] = mem
return mem
@staticmethod
def _turns(mem) -> int:
return len(mem.get_recent_messages()) if mem.has_recent_messages() else 0
def prepare(self, user: str, text: str, chat: bool) -> None:
from jarvis.reply.engine import prepare_intent
mem = self._memory(user)
p = _Prepared(text=text, turns=self._turns(mem), at=time.monotonic())
with self._lock:
self._prepared[user] = p
t0 = time.perf_counter()
try:
p.tools = Muted.run(prepare_intent, self.cfg, text, mem, chat, schedule_available=True,
notes_available=True, screen_available=True)
finally:
p.done.set()
debug_log(f"brain prepare user={user} picked={p.tools is not None} "
f"ms={(time.perf_counter() - t0) * 1000:.0f}", "llm")
def _prepared_tools(self, user: str, text: str) -> Optional[list]:
"""The pick prepared for about this text, waiting for one still being made; None if none fits."""
with self._lock:
p = self._prepared.pop(user, None)
if p is None or not similar(p.text, text) or time.monotonic() - p.at > PREPARED_MAX_AGE_S:
return None
if not p.done.wait(PREPARED_WAIT_S) or p.turns != self._turns(self._memory(user)):
return None
return p.tools
def _task_cfg(self):
from jarvis.tools.registry import BUILTIN_TOOLS
off = (*self.cfg.disabled_tools, *(t for t in BUILTIN_TOOLS if t not in TASK_TOOLS))
return dataclasses.replace(self.cfg, disabled_tools=tuple(dict.fromkeys(off)), mcps={})
def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None,
task: bool = False, notes: Optional[list] = None, notes_backend=None,
screen=None, trace: Optional[list] = None) -> Optional[str]:
from jarvis.memory.conversation import DialogueMemory
from jarvis.reply import turn_state
from jarvis.reply.engine import run_reply_engine
t0 = time.perf_counter()
# A task has its own short-term memory: it is not a turn of the person's conversation.
prepared = None if task else self._prepared_tools(user, text)
cfg = self._task_cfg() if task else self.cfg
memory = DialogueMemory(inactivity_timeout=self.cfg.dialogue_memory_timeout, max_interactions=20) \
if task else self._memory(user)
# legacy_memory=False: v1's profile, diary and graph belong to one user; several people
# share this assistant, so only the asker's scoped history may reach the prompt.
def turn():
# Tools with personal data (meals) read the asker from the turn state.
turn_state.reset()
turn_state.set_user(user)
turn_state.set_chat(None if task else chat)
turn_state.set_schedule(None if task else schedule)
turn_state.set_notes(None if task else notes_backend)
turn_state.set_screen(None if task else screen)
turn_state.set_trace(trace)
try:
return run_reply_engine(self.db, cfg, None, text, memory,
language=self.language, history=history, legacy_memory=False,
default_language=self.reply_language, compact_prompt=True,
intent_judge=True, prepared_tools=prepared,
person_notes=notes or None)
finally:
turn_state.reset()
answer = Muted.run(turn)
debug_log(f"brain reply user={user} ok={bool(answer)} task={task} prepared={prepared is not None} "
f"ms={(time.perf_counter() - t0) * 1000:.0f}", "llm")
return answer
def addressed(self, user: str, text: str, names: list[str]) -> Optional[bool]:
from .judges import is_addressed
return is_addressed(self.cfg.ollama_base_url, self.cfg.ollama_chat_model, names, text)
def rank(self, candidates: list[dict]) -> Optional[list[str]]:
from .judges import rank_requests
order = rank_requests(self.cfg.ollama_base_url, self.cfg.ollama_chat_model, [c["text"] for c in candidates])
return None if order is None else [candidates[i]["id"] for i in order]