""" Eval: seeing the screen (src/jarvis/tools/builtin/screen_view.spec.md). - "2번 모니터에 뭐 떠 있어?" calls screenView for monitor 2, and the answer reads the picture: the fake screen shows Notepad with "BANANA 42", which the model can only know from the picture. - When the app refuses (the person is not registered), the answer says it cannot see the screen and does not describe one. - A question that is not about the screen does not capture it. - "철수 화면 뭐 하고 있어?" asks for 철수's Discord screen share (target share, person 철수), and the answer reads that picture. The screen backend is a fake that returns a fixed picture; the tool, the reply loop and the picture transport (llm.py) run for real. Needs a model that reads pictures: EVAL_JUDGE_MODEL=gemma3:4b python -m pytest evals/test_screen_view_tool.py -v (Ollama's own API: its /v1 keeps a 4,096-token context, too small for the prompt and a picture) (the real v2 model: Qwen3.6 through Strata, EVAL_JUDGE_BASE_URL=http://:8080/v1; v2 brain setup: EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1) """ import base64 from pathlib import Path import pytest from conftest import requires_judge_llm PICTURE = base64.b64encode((Path(__file__).parent / "fixtures" / "screen_notepad_banana.jpg").read_bytes()).decode() class FakeScreen: def __init__(self, refuse=None): self.acts, self.refuse = [], refuse def act(self, action, **kw): self.acts.append((action, kw)) if self.refuse: return {"ok": False, "error": self.refuse} if action == "list": return {"ok": True, "item": {"monitors": [{"number": 1, "width": 2560, "height": 1440, "main": True}, {"number": 2, "width": 1280, "height": 720, "main": False}], "windows": [{"program": "notepad", "title": "Untitled - Notepad", "minimised": False}]}} what = "화면공유" if kw.get("target") == "share" else f"모니터 {kw.get('monitor', 2)}" return {"ok": True, "item": {"what": what, "width": 1280, "height": 720, "parts": 1}, "images": [PICTURE]} def _run(mock_config, eval_db, eval_dialogue_memory, query, screen): from helpers import JUDGE_BASE_URL, JUDGE_MODEL from jarvis.memory.history import HistoryContext from jarvis.reply import turn_state from jarvis.reply.engine import run_reply_engine mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL turn_state.reset() turn_state.set_user("100") turn_state.set_screen(screen) try: reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query, dialogue_memory=eval_dialogue_memory, history=HistoryContext(), legacy_memory=False, default_language="Korean") finally: turn_state.reset() print(f"\n screen [{JUDGE_MODEL}] {query!r}: acts={screen.acts} reply={(reply or '')[:200]!r}") return reply or "" @pytest.mark.eval @requires_judge_llm class TestScreenView: def test_the_answer_reads_the_monitor_it_was_asked_about(self, mock_config, eval_db, eval_dialogue_memory): screen = FakeScreen() reply = _run(mock_config, eval_db, eval_dialogue_memory, "2번 모니터에 뭐 떠 있어?", screen) captures = [kw for a, kw in screen.acts if a == "capture"] assert captures and captures[0].get("target") == "monitor" and captures[0].get("monitor") == 2, screen.acts assert "BANANA" in reply.upper() or "바나나" in reply or "42" in reply, reply def test_a_refusal_is_said_plainly_without_describing_a_screen(self, mock_config, eval_db, eval_dialogue_memory): screen = FakeScreen(refuse="PC 제어 화면에 등록된 사람만 화면을 볼 수 있습니다") reply = _run(mock_config, eval_db, eval_dialogue_memory, "지금 내 화면에 뭐 떠 있어?", screen) assert screen.acts, "the tool was asked" assert "BANANA" not in reply.upper() and "메모장" not in reply and "notepad" not in reply.lower(), reply assert any(w in reply for w in ("수 없", "수가 없", "등록", "권한", "못 ")), reply def test_a_question_not_about_the_screen_captures_nothing(self, mock_config, eval_db, eval_dialogue_memory): screen = FakeScreen() _run(mock_config, eval_db, eval_dialogue_memory, "고양이는 하루에 몇 시간 자?", screen) assert not any(a == "capture" for a, _ in screen.acts), screen.acts def test_someones_screen_share_is_asked_for_by_their_name(self, mock_config, eval_db, eval_dialogue_memory): screen = FakeScreen() reply = _run(mock_config, eval_db, eval_dialogue_memory, "철수 화면 뭐 하고 있어?", screen) captures = [kw for a, kw in screen.acts if a == "capture"] assert captures and captures[0].get("target") == "share" and "철수" in (captures[0].get("person") or ""), \ screen.acts assert "BANANA" in reply.upper() or "바나나" in reply or "42" in reply, reply