- screenView tool (screen_view.spec.md): whole screen, monitor by number, window by part of its title or program, or the list of monitors and windows; fast, normal or detailed pictures. - The app captures with Windows Graphics Capture (app/src/screen.rs, admin.spec.md "Screen viewing"), decides for the asker (only people registered for PC control), never picks, lists or shows excluded programs (pcControl.screenExclude, password managers by default), refuses minimised windows and black pictures, and records screen.view in the activity log. - Pictures ride on the tool message (ToolExecutionResult.images); llm.py sends them to OpenAI-compatible servers as image parts after the tool messages. They stay in that reply only: not in the tool carryover, logs or prompt dumps. - The LLM reads pictures only when screen viewing is available (--vision gpu normally, cpu in a game), measured apart for the placement; otherwise screenView is a disabled tool. - Worker: screen_action / screen_action_result, 4 MiB messages. - Eval evals/test_screen_view_tool.py; Windows check tests/screen_windows.rs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
87 lines
4.4 KiB
Python
87 lines
4.4 KiB
Python
"""
|
|
Eval: seeing the screen (src/jarvis/tools/builtin/screen_view.spec.md).
|
|
|
|
- "2번 모니터에 뭐 떠 있어?" calls screenView for monitor 2, and the answer reads the picture: the
|
|
fake screen shows Notepad with "BANANA 42", which the model can only know from the picture.
|
|
- When the app refuses (the person is not registered), the answer says it cannot see the screen and
|
|
does not describe one.
|
|
- A question that is not about the screen does not capture it.
|
|
|
|
The screen backend is a fake that returns a fixed picture; the tool, the reply loop and the picture
|
|
transport (llm.py) run for real. Needs a model that reads pictures:
|
|
EVAL_JUDGE_MODEL=gemma3:4b python -m pytest evals/test_screen_view_tool.py -v
|
|
(Ollama's own API: its /v1 keeps a 4,096-token context, too small for the prompt and a picture)
|
|
(the real v2 model: Qwen3.6 through Strata, EVAL_JUDGE_BASE_URL=http://<pc>:8080/v1; v2 brain setup:
|
|
EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1)
|
|
"""
|
|
|
|
import base64
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from conftest import requires_judge_llm
|
|
|
|
PICTURE = base64.b64encode((Path(__file__).parent / "fixtures" / "screen_notepad_banana.jpg").read_bytes()).decode()
|
|
|
|
|
|
class FakeScreen:
|
|
def __init__(self, refuse=None):
|
|
self.acts, self.refuse = [], refuse
|
|
|
|
def act(self, action, **kw):
|
|
self.acts.append((action, kw))
|
|
if self.refuse:
|
|
return {"ok": False, "error": self.refuse}
|
|
if action == "list":
|
|
return {"ok": True, "item": {"monitors": [{"number": 1, "width": 2560, "height": 1440, "main": True},
|
|
{"number": 2, "width": 1280, "height": 720, "main": False}],
|
|
"windows": [{"program": "notepad", "title": "Untitled - Notepad",
|
|
"minimised": False}]}}
|
|
return {"ok": True, "item": {"what": f"모니터 {kw.get('monitor', 2)}", "width": 1280, "height": 720,
|
|
"parts": 1}, "images": [PICTURE]}
|
|
|
|
|
|
def _run(mock_config, eval_db, eval_dialogue_memory, query, screen):
|
|
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
|
|
from jarvis.memory.history import HistoryContext
|
|
from jarvis.reply import turn_state
|
|
from jarvis.reply.engine import run_reply_engine
|
|
|
|
mock_config.ollama_base_url = JUDGE_BASE_URL
|
|
mock_config.ollama_chat_model = JUDGE_MODEL
|
|
turn_state.reset()
|
|
turn_state.set_user("100")
|
|
turn_state.set_screen(screen)
|
|
try:
|
|
reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
|
|
dialogue_memory=eval_dialogue_memory, history=HistoryContext(),
|
|
legacy_memory=False, default_language="Korean")
|
|
finally:
|
|
turn_state.reset()
|
|
print(f"\n screen [{JUDGE_MODEL}] {query!r}: acts={screen.acts} reply={(reply or '')[:200]!r}")
|
|
return reply or ""
|
|
|
|
|
|
@pytest.mark.eval
|
|
@requires_judge_llm
|
|
class TestScreenView:
|
|
def test_the_answer_reads_the_monitor_it_was_asked_about(self, mock_config, eval_db, eval_dialogue_memory):
|
|
screen = FakeScreen()
|
|
reply = _run(mock_config, eval_db, eval_dialogue_memory, "2번 모니터에 뭐 떠 있어?", screen)
|
|
captures = [kw for a, kw in screen.acts if a == "capture"]
|
|
assert captures and captures[0].get("target") == "monitor" and captures[0].get("monitor") == 2, screen.acts
|
|
assert "BANANA" in reply.upper() or "바나나" in reply or "42" in reply, reply
|
|
|
|
def test_a_refusal_is_said_plainly_without_describing_a_screen(self, mock_config, eval_db, eval_dialogue_memory):
|
|
screen = FakeScreen(refuse="PC 제어 화면에 등록된 사람만 화면을 볼 수 있습니다")
|
|
reply = _run(mock_config, eval_db, eval_dialogue_memory, "지금 내 화면에 뭐 떠 있어?", screen)
|
|
assert screen.acts, "the tool was asked"
|
|
assert "BANANA" not in reply.upper() and "메모장" not in reply and "notepad" not in reply.lower(), reply
|
|
assert any(w in reply for w in ("수 없", "수가 없", "등록", "권한", "못 ")), reply
|
|
|
|
def test_a_question_not_about_the_screen_captures_nothing(self, mock_config, eval_db, eval_dialogue_memory):
|
|
screen = FakeScreen()
|
|
_run(mock_config, eval_db, eval_dialogue_memory, "고양이는 하루에 몇 시간 자?", screen)
|
|
assert not any(a == "capture" for a, _ in screen.acts), screen.acts
|