Files
javis_bot/evals/test_screen_view_tool.py
javis-bot 844958d311 feat(v2): screenView sees someone's Discord screen share in its popped-out window
The share target checks that the person asked about shares their screen in
Jarvis's voice channel, finds Discord's popped-out window (title not ending in
Discord, the voice channel's name first) and, when there is none, asks for it in
the channel and looks again every second for 30 s. Screen actions now wait as
long as PC actions in the worker.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 13:43:31 +09:00

97 lines
5.1 KiB
Python

"""
Eval: seeing the screen (src/jarvis/tools/builtin/screen_view.spec.md).
- "2번 모니터에 뭐 떠 있어?" calls screenView for monitor 2, and the answer reads the picture: the
fake screen shows Notepad with "BANANA 42", which the model can only know from the picture.
- When the app refuses (the person is not registered), the answer says it cannot see the screen and
does not describe one.
- A question that is not about the screen does not capture it.
- "철수 화면 뭐 하고 있어?" asks for 철수's Discord screen share (target share, person 철수), and
the answer reads that picture.
The screen backend is a fake that returns a fixed picture; the tool, the reply loop and the picture
transport (llm.py) run for real. Needs a model that reads pictures:
EVAL_JUDGE_MODEL=gemma3:4b python -m pytest evals/test_screen_view_tool.py -v
(Ollama's own API: its /v1 keeps a 4,096-token context, too small for the prompt and a picture)
(the real v2 model: Qwen3.6 through Strata, EVAL_JUDGE_BASE_URL=http://<pc>:8080/v1; v2 brain setup:
EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1)
"""
import base64
from pathlib import Path
import pytest
from conftest import requires_judge_llm
PICTURE = base64.b64encode((Path(__file__).parent / "fixtures" / "screen_notepad_banana.jpg").read_bytes()).decode()
class FakeScreen:
def __init__(self, refuse=None):
self.acts, self.refuse = [], refuse
def act(self, action, **kw):
self.acts.append((action, kw))
if self.refuse:
return {"ok": False, "error": self.refuse}
if action == "list":
return {"ok": True, "item": {"monitors": [{"number": 1, "width": 2560, "height": 1440, "main": True},
{"number": 2, "width": 1280, "height": 720, "main": False}],
"windows": [{"program": "notepad", "title": "Untitled - Notepad",
"minimised": False}]}}
what = "화면공유" if kw.get("target") == "share" else f"모니터 {kw.get('monitor', 2)}"
return {"ok": True, "item": {"what": what, "width": 1280, "height": 720, "parts": 1}, "images": [PICTURE]}
def _run(mock_config, eval_db, eval_dialogue_memory, query, screen):
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
from jarvis.memory.history import HistoryContext
from jarvis.reply import turn_state
from jarvis.reply.engine import run_reply_engine
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
turn_state.reset()
turn_state.set_user("100")
turn_state.set_screen(screen)
try:
reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
dialogue_memory=eval_dialogue_memory, history=HistoryContext(),
legacy_memory=False, default_language="Korean")
finally:
turn_state.reset()
print(f"\n screen [{JUDGE_MODEL}] {query!r}: acts={screen.acts} reply={(reply or '')[:200]!r}")
return reply or ""
@pytest.mark.eval
@requires_judge_llm
class TestScreenView:
def test_the_answer_reads_the_monitor_it_was_asked_about(self, mock_config, eval_db, eval_dialogue_memory):
screen = FakeScreen()
reply = _run(mock_config, eval_db, eval_dialogue_memory, "2번 모니터에 뭐 떠 있어?", screen)
captures = [kw for a, kw in screen.acts if a == "capture"]
assert captures and captures[0].get("target") == "monitor" and captures[0].get("monitor") == 2, screen.acts
assert "BANANA" in reply.upper() or "바나나" in reply or "42" in reply, reply
def test_a_refusal_is_said_plainly_without_describing_a_screen(self, mock_config, eval_db, eval_dialogue_memory):
screen = FakeScreen(refuse="PC 제어 화면에 등록된 사람만 화면을 볼 수 있습니다")
reply = _run(mock_config, eval_db, eval_dialogue_memory, "지금 내 화면에 뭐 떠 있어?", screen)
assert screen.acts, "the tool was asked"
assert "BANANA" not in reply.upper() and "메모장" not in reply and "notepad" not in reply.lower(), reply
assert any(w in reply for w in ("수 없", "수가 없", "등록", "권한", "못 ")), reply
def test_a_question_not_about_the_screen_captures_nothing(self, mock_config, eval_db, eval_dialogue_memory):
screen = FakeScreen()
_run(mock_config, eval_db, eval_dialogue_memory, "고양이는 하루에 몇 시간 자?", screen)
assert not any(a == "capture" for a, _ in screen.acts), screen.acts
def test_someones_screen_share_is_asked_for_by_their_name(self, mock_config, eval_db, eval_dialogue_memory):
screen = FakeScreen()
reply = _run(mock_config, eval_db, eval_dialogue_memory, "철수 화면 뭐 하고 있어?", screen)
captures = [kw for a, kw in screen.acts if a == "capture"]
assert captures and captures[0].get("target") == "share" and "철수" in (captures[0].get("person") or ""), \
screen.acts
assert "BANANA" in reply.upper() or "바나나" in reply or "42" in reply, reply