feat: screen viewing, Jarvis sees the whole screen, a monitor or a window for registered people

- screenView tool (screen_view.spec.md): whole screen, monitor by number, window by part of its
  title or program, or the list of monitors and windows; fast, normal or detailed pictures.
- The app captures with Windows Graphics Capture (app/src/screen.rs, admin.spec.md "Screen
  viewing"), decides for the asker (only people registered for PC control), never picks, lists
  or shows excluded programs (pcControl.screenExclude, password managers by default), refuses
  minimised windows and black pictures, and records screen.view in the activity log.
- Pictures ride on the tool message (ToolExecutionResult.images); llm.py sends them to
  OpenAI-compatible servers as image parts after the tool messages. They stay in that reply only:
  not in the tool carryover, logs or prompt dumps.
- The LLM reads pictures only when screen viewing is available (--vision gpu normally, cpu in a
  game), measured apart for the placement; otherwise screenView is a disabled tool.
- Worker: screen_action / screen_action_result, 4 MiB messages.
- Eval evals/test_screen_view_tool.py; Windows check tests/screen_windows.rs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-15 17:02:59 +09:00
parent bd7b69d4f4
commit 53bccb620d
39 changed files with 1844 additions and 68 deletions

View File

@@ -0,0 +1,86 @@
"""
Eval: seeing the screen (src/jarvis/tools/builtin/screen_view.spec.md).
- "2번 모니터에 뭐 떠 있어?" calls screenView for monitor 2, and the answer reads the picture: the
fake screen shows Notepad with "BANANA 42", which the model can only know from the picture.
- When the app refuses (the person is not registered), the answer says it cannot see the screen and
does not describe one.
- A question that is not about the screen does not capture it.
The screen backend is a fake that returns a fixed picture; the tool, the reply loop and the picture
transport (llm.py) run for real. Needs a model that reads pictures:
EVAL_JUDGE_MODEL=gemma3:4b python -m pytest evals/test_screen_view_tool.py -v
(Ollama's own API: its /v1 keeps a 4,096-token context, too small for the prompt and a picture)
(the real v2 model: Qwen3.6 through Strata, EVAL_JUDGE_BASE_URL=http://<pc>:8080/v1; v2 brain setup:
EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1)
"""
import base64
from pathlib import Path
import pytest
from conftest import requires_judge_llm
PICTURE = base64.b64encode((Path(__file__).parent / "fixtures" / "screen_notepad_banana.jpg").read_bytes()).decode()
class FakeScreen:
def __init__(self, refuse=None):
self.acts, self.refuse = [], refuse
def act(self, action, **kw):
self.acts.append((action, kw))
if self.refuse:
return {"ok": False, "error": self.refuse}
if action == "list":
return {"ok": True, "item": {"monitors": [{"number": 1, "width": 2560, "height": 1440, "main": True},
{"number": 2, "width": 1280, "height": 720, "main": False}],
"windows": [{"program": "notepad", "title": "Untitled - Notepad",
"minimised": False}]}}
return {"ok": True, "item": {"what": f"모니터 {kw.get('monitor', 2)}", "width": 1280, "height": 720,
"parts": 1}, "images": [PICTURE]}
def _run(mock_config, eval_db, eval_dialogue_memory, query, screen):
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
from jarvis.memory.history import HistoryContext
from jarvis.reply import turn_state
from jarvis.reply.engine import run_reply_engine
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
turn_state.reset()
turn_state.set_user("100")
turn_state.set_screen(screen)
try:
reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
dialogue_memory=eval_dialogue_memory, history=HistoryContext(),
legacy_memory=False, default_language="Korean")
finally:
turn_state.reset()
print(f"\n screen [{JUDGE_MODEL}] {query!r}: acts={screen.acts} reply={(reply or '')[:200]!r}")
return reply or ""
@pytest.mark.eval
@requires_judge_llm
class TestScreenView:
def test_the_answer_reads_the_monitor_it_was_asked_about(self, mock_config, eval_db, eval_dialogue_memory):
screen = FakeScreen()
reply = _run(mock_config, eval_db, eval_dialogue_memory, "2번 모니터에 뭐 떠 있어?", screen)
captures = [kw for a, kw in screen.acts if a == "capture"]
assert captures and captures[0].get("target") == "monitor" and captures[0].get("monitor") == 2, screen.acts
assert "BANANA" in reply.upper() or "바나나" in reply or "42" in reply, reply
def test_a_refusal_is_said_plainly_without_describing_a_screen(self, mock_config, eval_db, eval_dialogue_memory):
screen = FakeScreen(refuse="PC 제어 화면에 등록된 사람만 화면을 볼 수 있습니다")
reply = _run(mock_config, eval_db, eval_dialogue_memory, "지금 내 화면에 뭐 떠 있어?", screen)
assert screen.acts, "the tool was asked"
assert "BANANA" not in reply.upper() and "메모장" not in reply and "notepad" not in reply.lower(), reply
assert any(w in reply for w in ("수 없", "수가 없", "등록", "권한", "못 ")), reply
def test_a_question_not_about_the_screen_captures_nothing(self, mock_config, eval_db, eval_dialogue_memory):
screen = FakeScreen()
_run(mock_config, eval_db, eval_dialogue_memory, "고양이는 하루에 몇 시간 자?", screen)
assert not any(a == "capture" for a, _ in screen.acts), screen.acts