feat: screen viewing, Jarvis sees the whole screen, a monitor or a window for registered people
- screenView tool (screen_view.spec.md): whole screen, monitor by number, window by part of its title or program, or the list of monitors and windows; fast, normal or detailed pictures. - The app captures with Windows Graphics Capture (app/src/screen.rs, admin.spec.md "Screen viewing"), decides for the asker (only people registered for PC control), never picks, lists or shows excluded programs (pcControl.screenExclude, password managers by default), refuses minimised windows and black pictures, and records screen.view in the activity log. - Pictures ride on the tool message (ToolExecutionResult.images); llm.py sends them to OpenAI-compatible servers as image parts after the tool messages. They stay in that reply only: not in the tool carryover, logs or prompt dumps. - The LLM reads pictures only when screen viewing is available (--vision gpu normally, cpu in a game), measured apart for the placement; otherwise screenView is a disabled tool. - Worker: screen_action / screen_action_result, 4 MiB messages. - Eval evals/test_screen_view_tool.py; Windows check tests/screen_windows.rs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
"""Backends the brain's schedule and personalNotes tools use during one reply (schedule.spec.md,
|
||||
personal_notes.spec.md).
|
||||
"""Backends the brain's schedule, personalNotes and screenView tools use during one reply
|
||||
(schedule.spec.md, personal_notes.spec.md, screen_view.spec.md).
|
||||
|
||||
The app keeps the schedules and the notes: every action goes to it as ``<kind>_action`` for the
|
||||
The app keeps the schedules and the notes and captures the screen: every action goes to it as ``<kind>_action`` for the
|
||||
person asking and waits for its ``<kind>_action_result``. Built per reply, on the event loop;
|
||||
``act`` is called from the brain's thread.
|
||||
"""
|
||||
@@ -17,7 +17,7 @@ class AppBridge:
|
||||
channel: Optional[str]):
|
||||
self._session = session
|
||||
self._loop = loop
|
||||
self._kind = kind # "schedule" or "note"
|
||||
self._kind = kind # "schedule", "note" or "screen"
|
||||
# Who asked and where: the app delivers schedules there (voice, or the chat channel asked in).
|
||||
self._origin = {"user": user, "via": via, **({"channel": channel} if channel else {})}
|
||||
|
||||
|
||||
@@ -18,14 +18,16 @@ from jarvis.memory.history import HistoryContext, Scope
|
||||
|
||||
class Brain(Protocol):
|
||||
def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None,
|
||||
task: bool = False, notes: Optional[list] = None, notes_backend=None) -> Optional[str]:
|
||||
task: bool = False, notes: Optional[list] = None, notes_backend=None,
|
||||
screen=None) -> Optional[str]:
|
||||
"""Blocking; called on a worker thread, one call per person at a time (a voice and a chat
|
||||
reply for different people may overlap). ``chat``: the chat backend for
|
||||
the chatChannels tool (voice_worker/chat_bridge.py), or None without registered channels.
|
||||
``schedule``: the schedule backend for the schedule tool (voice_worker/app_bridge.py).
|
||||
``task``: a scheduled task runs, with the read-only tools only (``TASK_TOOLS``).
|
||||
``notes``: the asker's standing instructions, put in the prompt; ``notes_backend``: the
|
||||
backend for the personalNotes tool (personal_notes.spec.md)."""
|
||||
backend for the personalNotes tool (personal_notes.spec.md); ``screen``: the backend for the
|
||||
screenView tool (screen_view.spec.md)."""
|
||||
|
||||
def addressed(self, user: str, text: str, names: list[str]) -> Optional[bool]:
|
||||
"""Blocking: was ``text`` said to the assistant (called ``names``)? None when it could not tell.
|
||||
@@ -270,7 +272,7 @@ class JarvisBrain:
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
p.tools = Muted.run(prepare_intent, self.cfg, text, mem, chat, schedule_available=True,
|
||||
notes_available=True)
|
||||
notes_available=True, screen_available=True)
|
||||
finally:
|
||||
p.done.set()
|
||||
debug_log(f"brain prepare user={user} picked={p.tools is not None} "
|
||||
@@ -293,7 +295,8 @@ class JarvisBrain:
|
||||
return dataclasses.replace(self.cfg, disabled_tools=tuple(dict.fromkeys(off)), mcps={})
|
||||
|
||||
def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None,
|
||||
task: bool = False, notes: Optional[list] = None, notes_backend=None) -> Optional[str]:
|
||||
task: bool = False, notes: Optional[list] = None, notes_backend=None,
|
||||
screen=None) -> Optional[str]:
|
||||
from jarvis.memory.conversation import DialogueMemory
|
||||
from jarvis.reply import turn_state
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
@@ -313,6 +316,7 @@ class JarvisBrain:
|
||||
turn_state.set_chat(None if task else chat)
|
||||
turn_state.set_schedule(None if task else schedule)
|
||||
turn_state.set_notes(None if task else notes_backend)
|
||||
turn_state.set_screen(None if task else screen)
|
||||
try:
|
||||
return run_reply_engine(self.db, cfg, None, text, memory,
|
||||
language=self.language, history=history, legacy_memory=False,
|
||||
|
||||
@@ -135,7 +135,7 @@ class VoiceWorker:
|
||||
raise ValueError("the voice worker only listens on a loopback address")
|
||||
from websockets.asyncio.server import serve
|
||||
await self.start()
|
||||
return await serve(self.handle, host, port, max_size=2 ** 20)
|
||||
return await serve(self.handle, host, port, max_size=4 * 2 ** 20) # pictures (screen_action_result)
|
||||
|
||||
async def handle(self, ws) -> None:
|
||||
try:
|
||||
@@ -354,6 +354,7 @@ class Session:
|
||||
"chat_channels": self._on_chat_channels, "chat_action_result": self._on_chat_action_result,
|
||||
"schedule_action_result": self._on_app_action_result,
|
||||
"note_action_result": self._on_app_action_result,
|
||||
"screen_action_result": self._on_app_action_result,
|
||||
"memory_config": self._on_memory_config, "reply_delivered": self._on_reply_delivered,
|
||||
"prepare": self._on_prepare, "llm": self._on_llm}.get(kind)
|
||||
if handler is None:
|
||||
@@ -523,7 +524,7 @@ class Session:
|
||||
fut.set_result(result)
|
||||
|
||||
async def app_action(self, kind: str, action: str, args: dict) -> dict:
|
||||
"""Ask the app to add, list or delete a schedule or a note (``kind``) and wait for its answer
|
||||
"""Ask the app to act on a schedule, a note or the screen (``kind``) and wait for its answer
|
||||
(or a timeout)."""
|
||||
aid = f"{self.worker.chat_action_prefix}-{self.sid}-{next(self._chat_ids)}"
|
||||
fut = asyncio.get_running_loop().create_future()
|
||||
@@ -549,6 +550,8 @@ class Session:
|
||||
result["item"] = msg["item"]
|
||||
if isinstance(msg.get("items"), list):
|
||||
result["items"] = [i for i in msg["items"] if isinstance(i, dict)]
|
||||
if isinstance(msg.get("images"), list): # screen pictures, base64 JPEG
|
||||
result["images"] = [i for i in msg["images"] if isinstance(i, str)]
|
||||
fut.set_result(result)
|
||||
|
||||
def _on_memory_config(self, msg: dict) -> None:
|
||||
@@ -820,13 +823,14 @@ class Session:
|
||||
origin = (user, via, chat_channel if via == "chat" else None)
|
||||
schedule = AppBridge(self, asyncio.get_running_loop(), "schedule", *origin)
|
||||
notes_backend = AppBridge(self, asyncio.get_running_loop(), "note", *origin)
|
||||
screen = AppBridge(self, asyncio.get_running_loop(), "screen", *origin)
|
||||
try:
|
||||
async with self._llm(rid if speak else None):
|
||||
w.answering += 1
|
||||
try:
|
||||
answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat,
|
||||
schedule=schedule, notes=notes or [],
|
||||
notes_backend=notes_backend)
|
||||
notes_backend=notes_backend, screen=screen)
|
||||
finally:
|
||||
w.answering -= 1
|
||||
if not w.answering and w.scheduler:
|
||||
|
||||
@@ -367,6 +367,22 @@ which keeps each person's standing instructions and sends the asker's with every
|
||||
- Every voice and chat reply gets a notes backend for its asker; ids and waiting are those of chat
|
||||
and schedule actions.
|
||||
|
||||
### Screen viewing (M7)
|
||||
|
||||
The brain's screenView tool (`src/jarvis/tools/builtin/screen_view.spec.md`) asks the app, which
|
||||
captures and decides who may see (`app/src/admin.spec.md`, "Screen viewing"):
|
||||
|
||||
```
|
||||
<- {"type": "screen_action", "id": "s1", "action": "capture|list", "user": "<id>", "via": "voice|chat",
|
||||
"channel": "...", "target": "screen|monitor|window", "monitor": 2, "window": "chrome", "detail": "fast|normal|detailed"}
|
||||
-> {"type": "screen_action_result", "id": "s1", "ok": true, "item": {...}, "images": ["<base64 JPEG>"]} // or ok false with "error"
|
||||
```
|
||||
|
||||
- Every voice and chat reply gets a screen backend for its asker (not scheduled tasks); ids and
|
||||
waiting are those of chat and schedule actions. `images` that are not a list of strings are
|
||||
dropped. The pictures stay in that one reply (screen_view.spec.md).
|
||||
- Messages may be up to 4 MiB (pictures), the WebSocket's limit.
|
||||
|
||||
### Background collection (M6)
|
||||
|
||||
With `JARVIS_COLLECT_PATH` set, the worker collects on its own thread, apart from the voice and the
|
||||
|
||||
Reference in New Issue
Block a user