344 lines
16 KiB
Python
344 lines
16 KiB
Python
"""The brain the voice worker hosts: memory settings and the jarvis reply engine behind a small
|
|
interface, so the worker logic is tested with a fake (voice_worker.spec.md, "History and brain")."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
import difflib
|
|
import sys
|
|
import unicodedata
|
|
import threading
|
|
import time
|
|
from dataclasses import dataclass
|
|
from typing import Optional, Protocol
|
|
|
|
from .log import debug_log, notice
|
|
from jarvis.memory.history import HistoryContext, Scope
|
|
|
|
|
|
class Brain(Protocol):
|
|
def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None,
|
|
task: bool = False, notes: Optional[list] = None, notes_backend=None,
|
|
screen=None, trace: Optional[list] = None) -> Optional[str]:
|
|
"""Blocking; called on a worker thread, one call per person at a time (a voice and a chat
|
|
reply for different people may overlap). ``chat``: the chat backend for
|
|
the chatChannels tool (voice_worker/chat_bridge.py), or None without registered channels.
|
|
``schedule``: the schedule backend for the schedule tool (voice_worker/app_bridge.py).
|
|
``task``: a scheduled task runs, with the read-only tools only (``TASK_TOOLS``).
|
|
``notes``: the asker's standing instructions, put in the prompt; ``notes_backend``: the
|
|
backend for the personalNotes tool (personal_notes.spec.md); ``screen``: the backend for the
|
|
screenView tool (screen_view.spec.md); ``trace``: a list the engine adds the steps of
|
|
this answer to (history.spec.md, "Traces")."""
|
|
|
|
def addressed(self, user: str, text: str, names: list[str]) -> Optional[bool]:
|
|
"""Blocking: was ``text`` said to the assistant (called ``names``)? None when it could not tell.
|
|
Runs on its own thread, also while a reply runs."""
|
|
|
|
def rank(self, candidates: list[dict]) -> Optional[list[str]]:
|
|
"""Blocking: candidate ids (``{"id", "user", "text"}``) most important first; None on failure.
|
|
Runs on its own thread, also while a reply runs."""
|
|
|
|
def prepare(self, user: str, text: str, chat: bool) -> None:
|
|
"""Blocking: get ready to answer ``text`` (a partial transcript of a call still being said),
|
|
so the reply to about the same text starts further along. ``chat``: chat channels are
|
|
registered. Runs on its own thread, also while a reply runs. Optional."""
|
|
|
|
|
|
@dataclass
|
|
class MemorySettings:
|
|
chat: bool = True
|
|
others: bool = True
|
|
off: bool = False
|
|
since_days: Optional[float] = None
|
|
recent_minutes: float = 30
|
|
recent_records: int = 20
|
|
matches: int = 5
|
|
budget_tokens: float = 4000
|
|
|
|
_BOOLS = ("chat", "others", "off")
|
|
_NUMBERS = {"recent_minutes": (0, 24 * 60), "recent_records": (0, 200), "matches": (0, 20),
|
|
"budget_tokens": (0, 100_000)}
|
|
|
|
def update(self, changes: dict) -> None:
|
|
"""Apply ``changes`` atomically; raises ValueError naming the first bad field."""
|
|
new = dict(self.__dict__)
|
|
for key, value in changes.items():
|
|
if key in self._BOOLS:
|
|
if not isinstance(value, bool):
|
|
raise ValueError(f"{key} must be true or false")
|
|
elif key == "since_days":
|
|
if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float))
|
|
or value <= 0):
|
|
raise ValueError("since_days must be a positive number or null")
|
|
elif key in self._NUMBERS:
|
|
lo, hi = self._NUMBERS[key]
|
|
if isinstance(value, bool) or not isinstance(value, (int, float)) or not lo <= value <= hi:
|
|
raise ValueError(f"{key} must be a number from {lo} to {hi}")
|
|
if key in ("recent_records", "matches"):
|
|
value = int(value)
|
|
else:
|
|
raise ValueError(f"unknown memory setting: {key}")
|
|
new[key] = value
|
|
self.__dict__.update(new)
|
|
|
|
def scope(self, now: float) -> Scope:
|
|
since = now - self.since_days * 86400 if self.since_days else None
|
|
return Scope(chat=self.chat, others=self.others, since=since, off=self.off)
|
|
|
|
|
|
class Muted:
|
|
"""Stand-in for sys.stdout/sys.stderr that drops every write while a brain reply runs.
|
|
|
|
The v1 reply engine and its tools print the question, the answer, tool arguments, URLs and
|
|
memory snippets (``print`` and ``debug_log``, also from helper threads such as the web fetch
|
|
pool); the worker's logs must never carry such text. Replies may overlap (a voice and a chat
|
|
lane); output stays muted while any of them runs. The worker's own log (``voice_worker.log``)
|
|
writes to the original stderr and is not muted.
|
|
"""
|
|
|
|
_lock = threading.Lock()
|
|
_depth = 0
|
|
|
|
def __init__(self, target):
|
|
self._target = target
|
|
|
|
def write(self, data):
|
|
if Muted._depth:
|
|
return len(data)
|
|
return self._target.write(data)
|
|
|
|
def __getattr__(self, name):
|
|
return getattr(self._target, name)
|
|
|
|
@classmethod
|
|
def run(cls, fn, *args, **kwargs):
|
|
with cls._lock:
|
|
for name in ("stdout", "stderr"):
|
|
stream = getattr(sys, name)
|
|
if not isinstance(stream, cls):
|
|
setattr(sys, name, cls(stream))
|
|
cls._depth += 1
|
|
try:
|
|
return fn(*args, **kwargs)
|
|
finally:
|
|
with cls._lock:
|
|
cls._depth -= 1
|
|
|
|
|
|
# A prepared intent is used for a reply whose text is at least this close to the text it was
|
|
# prepared for (letters and digits only): the last partial often lacks a final syllable or two.
|
|
PREPARED_SIMILARITY = 0.85
|
|
# ... and only this soon after it was prepared, and at most this long is waited for one in flight.
|
|
PREPARED_MAX_AGE_S = 30.0
|
|
PREPARED_WAIT_S = 5.0
|
|
|
|
|
|
def _letters(text: str) -> str:
|
|
return "".join(c for c in unicodedata.normalize("NFKD", text.lower()) if c.isalnum())
|
|
|
|
|
|
def similar(a: str, b: str) -> bool:
|
|
x, y = _letters(a), _letters(b)
|
|
return bool(x and y) and difflib.SequenceMatcher(None, x, y).ratio() >= PREPARED_SIMILARITY
|
|
|
|
|
|
@dataclass
|
|
class _Prepared:
|
|
text: str
|
|
turns: int # the asker's dialogue memory size it was made with
|
|
at: float
|
|
done: threading.Event = dataclasses.field(default_factory=threading.Event)
|
|
tools: Optional[list] = None # the intent judge's pick; None: none made
|
|
|
|
|
|
# The app passes the MCP servers' environment values (API keys, kept in its encrypted secrets)
|
|
# here, as JSON ``{server: {NAME: value}}``, never in the config file.
|
|
MCP_ENV_VAR = "JARVIS_MCP_ENV"
|
|
# v1's host tools: the PC owner's home folder (read, write, delete) and screen. In v2 anyone in the
|
|
# voice channel or a registered chat channel asks, so they are never offered (M7 brings screen
|
|
# viewing with its own permissions).
|
|
V2_HOST_TOOLS = ("localFiles", "screenshot")
|
|
# The only tools a scheduled task may use: it runs with nobody there to ask, so it only looks
|
|
# things up (schedule.spec.md, "Tasks"). No MCP tools either: what they do is not known.
|
|
TASK_TOOLS = ("getWeather", "webSearch", "fetchWebPage", "fetchMeals", "collectedNews", "toolSearchTool")
|
|
|
|
|
|
def take_mcp_env() -> dict:
|
|
"""The app's MCP environment, removed from this process's environment so no child process
|
|
(MCP servers included) inherits another server's secrets."""
|
|
import json
|
|
import os
|
|
|
|
raw = os.environ.pop(MCP_ENV_VAR, None)
|
|
if not raw:
|
|
return {}
|
|
try:
|
|
env = json.loads(raw)
|
|
except ValueError:
|
|
notice("Warning: the MCP environment from the app is not JSON; ignored", "llm")
|
|
return {}
|
|
if not isinstance(env, dict):
|
|
return {}
|
|
return {str(k): {str(n): str(v) for n, v in vars_.items()} for k, vars_ in env.items()
|
|
if isinstance(vars_, dict)}
|
|
|
|
|
|
def _with_mcp_env(mcps: dict, env: dict) -> dict:
|
|
out = {}
|
|
for name, server in mcps.items():
|
|
server = dict(server)
|
|
if env.get(name):
|
|
server["env"] = {**(server.get("env") or {}), **env[name]}
|
|
out[name] = server
|
|
return out
|
|
|
|
|
|
def _connect_mcp(mcps: dict) -> None:
|
|
"""Discover the MCP servers' tools into the registry's cache (the reply engine offers them
|
|
from there). Runs on its own thread so a slow server does not hold up the voice."""
|
|
from jarvis.tools.registry import initialize_mcp_tools
|
|
|
|
t0 = time.perf_counter()
|
|
try:
|
|
tools, errors = initialize_mcp_tools(mcps, verbose=False)
|
|
except Exception as e: # noqa: BLE001 - the brain answers without MCP tools
|
|
notice(f"Warning: MCP servers could not be connected: {type(e).__name__}", "llm")
|
|
return
|
|
for name in mcps:
|
|
if name in errors:
|
|
notice(f"Warning: MCP server {name} failed: {errors[name]}", "llm")
|
|
else:
|
|
notice(f"MCP server {name}: {sum(1 for t in tools if t.startswith(f'{name}__'))} tool(s)", "llm")
|
|
debug_log(f"mcp ready: {len(tools)} tool(s) ms={(time.perf_counter() - t0) * 1000:.0f}", "llm")
|
|
|
|
|
|
class JarvisBrain:
|
|
"""The existing jarvis reply engine with the user's history context.
|
|
|
|
Uses the jarvis settings (``ollama_base_url`` is Strata's ``/v1`` in v2) and its database for
|
|
the long-term memory it already keeps. Each user has their own short-term dialogue memory so
|
|
tool carry-over follows the person who asked.
|
|
"""
|
|
|
|
def __init__(self, language: Optional[str] = "ko", reply_language: Optional[str] = "Korean",
|
|
mcp_env: Optional[dict] = None):
|
|
from jarvis.config import load_settings
|
|
from jarvis.memory.db import Database
|
|
|
|
# v1's tts_engine names the v1 desktop voice (English-only Piper by default), which makes the
|
|
# engine lock replies to English. v2 speaks through the worker's TTS and chat answers are
|
|
# not spoken; the brain answers in ``reply_language`` (an OUTPUT_LANGUAGE lock still wins).
|
|
# stream_browser: v1's screen-share browser (controlBrowser, browseAndPlay, setBroadcast and
|
|
# browser-first web search) needs v1's Node bot; v2 has no on-screen browser until M6.
|
|
# disabled_tools: the app's feature switches plus v1's host tools (V2_HOST_TOOLS).
|
|
base = load_settings()
|
|
self.cfg = dataclasses.replace(
|
|
base, tts_engine="melo", stream_browser=False,
|
|
disabled_tools=tuple(dict.fromkeys((*base.disabled_tools, *V2_HOST_TOOLS))),
|
|
mcps=_with_mcp_env(base.mcps, take_mcp_env() if mcp_env is None else mcp_env))
|
|
self.db = Database(self.cfg.db_path, self.cfg.sqlite_vss_path)
|
|
# ``language``: the ISO code tools get (e.g. the Wikipedia host); ``reply_language``: replies
|
|
# come in it unless the asker explicitly asks for another language (None: the asker's).
|
|
self.language = language
|
|
self.reply_language = reply_language
|
|
self._memories: dict = {}
|
|
self._prepared: dict[str, _Prepared] = {}
|
|
self._lock = threading.Lock()
|
|
debug_log(f"brain ready: llm={self.cfg.ollama_base_url} model={self.cfg.ollama_chat_model}", "llm")
|
|
if self.cfg.mcps:
|
|
threading.Thread(target=_connect_mcp, args=(self.cfg.mcps,), name="mcp-connect",
|
|
daemon=True).start()
|
|
|
|
def _memory(self, user: str):
|
|
from jarvis.memory.conversation import DialogueMemory
|
|
|
|
with self._lock:
|
|
mem = self._memories.get(user)
|
|
if mem is None:
|
|
mem = DialogueMemory(inactivity_timeout=self.cfg.dialogue_memory_timeout, max_interactions=20)
|
|
self._memories[user] = mem
|
|
return mem
|
|
|
|
@staticmethod
|
|
def _turns(mem) -> int:
|
|
return len(mem.get_recent_messages()) if mem.has_recent_messages() else 0
|
|
|
|
def prepare(self, user: str, text: str, chat: bool) -> None:
|
|
from jarvis.reply.engine import prepare_intent
|
|
|
|
mem = self._memory(user)
|
|
p = _Prepared(text=text, turns=self._turns(mem), at=time.monotonic())
|
|
with self._lock:
|
|
self._prepared[user] = p
|
|
t0 = time.perf_counter()
|
|
try:
|
|
p.tools = Muted.run(prepare_intent, self.cfg, text, mem, chat, schedule_available=True,
|
|
notes_available=True, screen_available=True)
|
|
finally:
|
|
p.done.set()
|
|
debug_log(f"brain prepare user={user} picked={p.tools is not None} "
|
|
f"ms={(time.perf_counter() - t0) * 1000:.0f}", "llm")
|
|
|
|
def _prepared_tools(self, user: str, text: str) -> Optional[list]:
|
|
"""The pick prepared for about this text, waiting for one still being made; None if none fits."""
|
|
with self._lock:
|
|
p = self._prepared.pop(user, None)
|
|
if p is None or not similar(p.text, text) or time.monotonic() - p.at > PREPARED_MAX_AGE_S:
|
|
return None
|
|
if not p.done.wait(PREPARED_WAIT_S) or p.turns != self._turns(self._memory(user)):
|
|
return None
|
|
return p.tools
|
|
|
|
def _task_cfg(self):
|
|
from jarvis.tools.registry import BUILTIN_TOOLS
|
|
|
|
off = (*self.cfg.disabled_tools, *(t for t in BUILTIN_TOOLS if t not in TASK_TOOLS))
|
|
return dataclasses.replace(self.cfg, disabled_tools=tuple(dict.fromkeys(off)), mcps={})
|
|
|
|
def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None,
|
|
task: bool = False, notes: Optional[list] = None, notes_backend=None,
|
|
screen=None, trace: Optional[list] = None) -> Optional[str]:
|
|
from jarvis.memory.conversation import DialogueMemory
|
|
from jarvis.reply import turn_state
|
|
from jarvis.reply.engine import run_reply_engine
|
|
|
|
t0 = time.perf_counter()
|
|
# A task has its own short-term memory: it is not a turn of the person's conversation.
|
|
prepared = None if task else self._prepared_tools(user, text)
|
|
cfg = self._task_cfg() if task else self.cfg
|
|
memory = DialogueMemory(inactivity_timeout=self.cfg.dialogue_memory_timeout, max_interactions=20) \
|
|
if task else self._memory(user)
|
|
# legacy_memory=False: v1's profile, diary and graph belong to one user; several people
|
|
# share this assistant, so only the asker's scoped history may reach the prompt.
|
|
def turn():
|
|
# Tools with personal data (meals) read the asker from the turn state.
|
|
turn_state.reset()
|
|
turn_state.set_user(user)
|
|
turn_state.set_chat(None if task else chat)
|
|
turn_state.set_schedule(None if task else schedule)
|
|
turn_state.set_notes(None if task else notes_backend)
|
|
turn_state.set_screen(None if task else screen)
|
|
turn_state.set_trace(trace)
|
|
try:
|
|
return run_reply_engine(self.db, cfg, None, text, memory,
|
|
language=self.language, history=history, legacy_memory=False,
|
|
default_language=self.reply_language, compact_prompt=True,
|
|
intent_judge=True, prepared_tools=prepared,
|
|
person_notes=notes or None)
|
|
finally:
|
|
turn_state.reset()
|
|
|
|
answer = Muted.run(turn)
|
|
debug_log(f"brain reply user={user} ok={bool(answer)} task={task} prepared={prepared is not None} "
|
|
f"ms={(time.perf_counter() - t0) * 1000:.0f}", "llm")
|
|
return answer
|
|
|
|
def addressed(self, user: str, text: str, names: list[str]) -> Optional[bool]:
|
|
from .judges import is_addressed
|
|
return is_addressed(self.cfg.ollama_base_url, self.cfg.ollama_chat_model, names, text)
|
|
|
|
def rank(self, candidates: list[dict]) -> Optional[list[str]]:
|
|
from .judges import rank_requests
|
|
order = rank_requests(self.cfg.ollama_base_url, self.cfg.ollama_chat_model, [c["text"] for c in candidates])
|
|
return None if order is None else [candidates[i]["id"] for i in order]
|