The share target checks that the person asked about shares their screen in Jarvis's voice channel, finds Discord's popped-out window (title not ending in Discord, the voice channel's name first) and, when there is none, asks for it in the channel and looks again every second for 30 s. Screen actions now wait as long as PC actions in the worker. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
97 lines
5.1 KiB
Python
97 lines
5.1 KiB
Python
"""
|
|
Eval: seeing the screen (src/jarvis/tools/builtin/screen_view.spec.md).
|
|
|
|
- "2번 모니터에 뭐 떠 있어?" calls screenView for monitor 2, and the answer reads the picture: the
|
|
fake screen shows Notepad with "BANANA 42", which the model can only know from the picture.
|
|
- When the app refuses (the person is not registered), the answer says it cannot see the screen and
|
|
does not describe one.
|
|
- A question that is not about the screen does not capture it.
|
|
- "철수 화면 뭐 하고 있어?" asks for 철수's Discord screen share (target share, person 철수), and
|
|
the answer reads that picture.
|
|
|
|
The screen backend is a fake that returns a fixed picture; the tool, the reply loop and the picture
|
|
transport (llm.py) run for real. Needs a model that reads pictures:
|
|
EVAL_JUDGE_MODEL=gemma3:4b python -m pytest evals/test_screen_view_tool.py -v
|
|
(Ollama's own API: its /v1 keeps a 4,096-token context, too small for the prompt and a picture)
|
|
(the real v2 model: Qwen3.6 through Strata, EVAL_JUDGE_BASE_URL=http://<pc>:8080/v1; v2 brain setup:
|
|
EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1)
|
|
"""
|
|
|
|
import base64
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from conftest import requires_judge_llm
|
|
|
|
PICTURE = base64.b64encode((Path(__file__).parent / "fixtures" / "screen_notepad_banana.jpg").read_bytes()).decode()
|
|
|
|
|
|
class FakeScreen:
|
|
def __init__(self, refuse=None):
|
|
self.acts, self.refuse = [], refuse
|
|
|
|
def act(self, action, **kw):
|
|
self.acts.append((action, kw))
|
|
if self.refuse:
|
|
return {"ok": False, "error": self.refuse}
|
|
if action == "list":
|
|
return {"ok": True, "item": {"monitors": [{"number": 1, "width": 2560, "height": 1440, "main": True},
|
|
{"number": 2, "width": 1280, "height": 720, "main": False}],
|
|
"windows": [{"program": "notepad", "title": "Untitled - Notepad",
|
|
"minimised": False}]}}
|
|
what = "화면공유" if kw.get("target") == "share" else f"모니터 {kw.get('monitor', 2)}"
|
|
return {"ok": True, "item": {"what": what, "width": 1280, "height": 720, "parts": 1}, "images": [PICTURE]}
|
|
|
|
|
|
def _run(mock_config, eval_db, eval_dialogue_memory, query, screen):
|
|
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
|
|
from jarvis.memory.history import HistoryContext
|
|
from jarvis.reply import turn_state
|
|
from jarvis.reply.engine import run_reply_engine
|
|
|
|
mock_config.ollama_base_url = JUDGE_BASE_URL
|
|
mock_config.ollama_chat_model = JUDGE_MODEL
|
|
turn_state.reset()
|
|
turn_state.set_user("100")
|
|
turn_state.set_screen(screen)
|
|
try:
|
|
reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
|
|
dialogue_memory=eval_dialogue_memory, history=HistoryContext(),
|
|
legacy_memory=False, default_language="Korean")
|
|
finally:
|
|
turn_state.reset()
|
|
print(f"\n screen [{JUDGE_MODEL}] {query!r}: acts={screen.acts} reply={(reply or '')[:200]!r}")
|
|
return reply or ""
|
|
|
|
|
|
@pytest.mark.eval
|
|
@requires_judge_llm
|
|
class TestScreenView:
|
|
def test_the_answer_reads_the_monitor_it_was_asked_about(self, mock_config, eval_db, eval_dialogue_memory):
|
|
screen = FakeScreen()
|
|
reply = _run(mock_config, eval_db, eval_dialogue_memory, "2번 모니터에 뭐 떠 있어?", screen)
|
|
captures = [kw for a, kw in screen.acts if a == "capture"]
|
|
assert captures and captures[0].get("target") == "monitor" and captures[0].get("monitor") == 2, screen.acts
|
|
assert "BANANA" in reply.upper() or "바나나" in reply or "42" in reply, reply
|
|
|
|
def test_a_refusal_is_said_plainly_without_describing_a_screen(self, mock_config, eval_db, eval_dialogue_memory):
|
|
screen = FakeScreen(refuse="PC 제어 화면에 등록된 사람만 화면을 볼 수 있습니다")
|
|
reply = _run(mock_config, eval_db, eval_dialogue_memory, "지금 내 화면에 뭐 떠 있어?", screen)
|
|
assert screen.acts, "the tool was asked"
|
|
assert "BANANA" not in reply.upper() and "메모장" not in reply and "notepad" not in reply.lower(), reply
|
|
assert any(w in reply for w in ("수 없", "수가 없", "등록", "권한", "못 ")), reply
|
|
|
|
def test_a_question_not_about_the_screen_captures_nothing(self, mock_config, eval_db, eval_dialogue_memory):
|
|
screen = FakeScreen()
|
|
_run(mock_config, eval_db, eval_dialogue_memory, "고양이는 하루에 몇 시간 자?", screen)
|
|
assert not any(a == "capture" for a, _ in screen.acts), screen.acts
|
|
|
|
def test_someones_screen_share_is_asked_for_by_their_name(self, mock_config, eval_db, eval_dialogue_memory):
|
|
screen = FakeScreen()
|
|
reply = _run(mock_config, eval_db, eval_dialogue_memory, "철수 화면 뭐 하고 있어?", screen)
|
|
captures = [kw for a, kw in screen.acts if a == "capture"]
|
|
assert captures and captures[0].get("target") == "share" and "철수" in (captures[0].get("person") or ""), \
|
|
screen.acts
|
|
assert "BANANA" in reply.upper() or "바나나" in reply or "42" in reply, reply
|