feat: screen viewing, Jarvis sees the whole screen, a monitor or a window for registered people

- screenView tool (screen_view.spec.md): whole screen, monitor by number, window by part of its
  title or program, or the list of monitors and windows; fast, normal or detailed pictures.
- The app captures with Windows Graphics Capture (app/src/screen.rs, admin.spec.md "Screen
  viewing"), decides for the asker (only people registered for PC control), never picks, lists
  or shows excluded programs (pcControl.screenExclude, password managers by default), refuses
  minimised windows and black pictures, and records screen.view in the activity log.
- Pictures ride on the tool message (ToolExecutionResult.images); llm.py sends them to
  OpenAI-compatible servers as image parts after the tool messages. They stay in that reply only:
  not in the tool carryover, logs or prompt dumps.
- The LLM reads pictures only when screen viewing is available (--vision gpu normally, cpu in a
  game), measured apart for the placement; otherwise screenView is a disabled tool.
- Worker: screen_action / screen_action_result, 4 MiB messages.
- Eval evals/test_screen_view_tool.py; Windows check tests/screen_windows.rs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-15 17:02:59 +09:00
parent bd7b69d4f4
commit 53bccb620d
39 changed files with 1844 additions and 68 deletions

71
app/Cargo.lock generated
View File

@@ -942,7 +942,7 @@ dependencies = [
"libc",
"log",
"rustversion",
"windows",
"windows 0.48.0",
]
[[package]]
@@ -1491,6 +1491,8 @@ dependencies = [
"chrono",
"fs4",
"futures-util",
"jpeg-decoder",
"jpeg-encoder",
"libc",
"opus2",
"rand 0.9.5",
@@ -1504,10 +1506,26 @@ dependencies = [
"tokio",
"tokio-tungstenite 0.26.2",
"unicode-normalization",
"windows 0.62.2",
"windows-sys 0.61.2",
"zip",
]
[[package]]
name = "jpeg-decoder"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07"
dependencies = [
"rayon",
]
[[package]]
name = "jpeg-encoder"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b454d911ac55068f53495488d8ccd0646eaa540c033a28ee15b07838afafb01f"
[[package]]
name = "js-sys"
version = "0.3.107"
@@ -4257,6 +4275,27 @@ dependencies = [
"windows-targets 0.48.5",
]
[[package]]
name = "windows"
version = "0.62.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580"
dependencies = [
"windows-collections",
"windows-core",
"windows-future",
"windows-numerics",
]
[[package]]
name = "windows-collections"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610"
dependencies = [
"windows-core",
]
[[package]]
name = "windows-core"
version = "0.62.2"
@@ -4270,6 +4309,17 @@ dependencies = [
"windows-strings",
]
[[package]]
name = "windows-future"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb"
dependencies = [
"windows-core",
"windows-link",
"windows-threading",
]
[[package]]
name = "windows-implement"
version = "0.60.2"
@@ -4298,6 +4348,16 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
[[package]]
name = "windows-numerics"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26"
dependencies = [
"windows-core",
"windows-link",
]
[[package]]
name = "windows-result"
version = "0.4.1"
@@ -4365,6 +4425,15 @@ dependencies = [
"windows_x86_64_msvc 0.52.6",
]
[[package]]
name = "windows-threading"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37"
dependencies = [
"windows-link",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.48.5"

View File

@@ -34,6 +34,10 @@ sha2 = "0.10"
chrono = { version = "0.4", default-features = false, features = ["clock", "std"] }
zip = { version = "9", default-features = false, features = ["deflate"] }
unicode-normalization = "0.1"
jpeg-encoder = "0.6"
[dev-dependencies]
jpeg-decoder = "0.3"
[[test]]
name = "vdisplay_elevated"
@@ -45,7 +49,8 @@ codegen-units = 4
strip = "symbols"
[target.'cfg(windows)'.dependencies]
windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Globalization", "Win32_Security_Cryptography", "Win32_System_SystemInformation", "Win32_Devices_DeviceAndDriverInstallation", "Win32_Graphics_Gdi", "Win32_UI_WindowsAndMessaging", "Win32_System_Registry", "Win32_UI_Shell", "Win32_System_Threading", "Win32_System_Performance", "Win32_System_Diagnostics_ToolHelp"] }
windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Globalization", "Win32_Security_Cryptography", "Win32_System_SystemInformation", "Win32_Devices_DeviceAndDriverInstallation", "Win32_Graphics_Gdi", "Win32_UI_WindowsAndMessaging", "Win32_System_Registry", "Win32_UI_Shell", "Win32_System_Threading", "Win32_System_Performance", "Win32_System_Diagnostics_ToolHelp", "Win32_Graphics_Dwm"] }
windows = { version = "0.62", features = ["Foundation", "Graphics", "Graphics_Capture", "Graphics_DirectX", "Graphics_DirectX_Direct3D11", "Win32_Foundation", "Win32_Graphics_Gdi", "Win32_Graphics_Direct3D", "Win32_Graphics_Direct3D11", "Win32_Graphics_Dxgi", "Win32_Graphics_Dxgi_Common", "Win32_System_WinRT", "Win32_System_WinRT_Direct3D11", "Win32_System_WinRT_Graphics_Capture"] }
[target.'cfg(unix)'.dependencies]
libc = "0.2"

View File

@@ -65,10 +65,10 @@ manages the MCP servers it connects.
`browserShare` (봇용 크롬창 화면공유 선택 표시, the bot browser on the virtual monitor, below
"Virtual monitor"), off by default; `schedule` (reminders and scheduled tasks, below
"Schedules", and the brain tool `schedule`), on by default; `notes` (personal notes, below
"Personal notes", and the brain tool `personalNotes`), on by default. The tools of a feature that
"Personal notes", and the brain tool `personalNotes`), on by default; `screen` (화면 보기, below
"Screen viewing", and the brain tool `screenView`), on by default. The tools of a feature that
is off go into the brain's `disabled_tools`: they are neither offered nor run
(`voice_worker.spec.md`, "Tools"). The other features of the 기능 screen (PC control, screen
viewing) come with their milestone (M7).
(`voice_worker.spec.md`, "Tools"). PC control comes with its milestone (M7).
- `mcpServers` (new list): each `{name, command, args, env, enabled}`. `name` is 1-32 letters,
digits, `-` or `_` without `__` (the brain names a server's tools `<name>__<tool>`), not listed
twice; `command` is not empty; `args` a list of strings; `env` the names of the environment
@@ -162,6 +162,42 @@ decision 2026-11-14). What Jarvis may do on this PC and for whom; the PC control
- The PC 제어 screen edits the section and shows, read-only, what is never done and each registered
chat channel's send and delete switches (changed on the 채팅 채널 screen).
## Screen viewing (`app/src/screen.rs`)
`docs/v2/pc-features.md` ("화면 보기", "화면 질문을 빠르게"). The brain's `screenView` tool
(`src/jarvis/tools/builtin/screen_view.spec.md`) sees all monitors as one picture, one monitor, or
one window; the app captures and decides who may.
- **Available** only on Windows with `features.screen` on, someone in `pcControl.allowedUsers` and
`pcControl.view` not `never`. Otherwise `screenView` is in the brain's `disabled_tools` and the
LLM reads no pictures (`--vision off`, below "LLM"). Settings apply on the next start.
- **Who**: every `screen_action` (`voice_worker.spec.md`, "Screen viewing") is decided for its
asker by `permissions::decide(View)`: people not registered are refused ("PC 제어 화면에 등록된
사람만 화면을 볼 수 있습니다"), and so is viewing at `ask` until the PC control tool brings the
next-message consent (M7, next step). Viewing works while an anti-cheat game runs (it only reads).
- **Capture**: Windows Graphics Capture (D3D11, no cursor, no yellow border where Windows allows
it), one session per request, the first frame within 2 s. Monitors are numbered as Windows
numbers them (`\\.\DISPLAY<n>`); the bot browser's virtual monitor ("Virtual monitor") is left out
of the list and of the whole screen. `screen` puts all monitors in one picture as they are placed;
`window` takes the visible top-level window whose title or program name contains the text
(case-insensitive; the one nearest the front when several do), also when others cover it; a
minimised window is refused ("최소화되어 있어 볼 수 없습니다").
- **Exclusions**: `pcControl.screenExclude` (program file names, ".exe" may be left out; default
`1Password`, `Bitwarden`, `KeePass`, `KeePassXC`). Their windows are never captured or listed, and
in a monitor or whole-screen picture their area is painted black before anything leaves the
capture.
- **Pictures**: scaled down (area average) so the long side is at most 768 px (`fast`), 1280 px
(`normal`) or, for `detailed`, cut in four parts of at most 1280 px each (a picture whose long side
is at most 1600 px stays one part at full size); JPEG quality 80. At most 3 MiB of pictures per
answer, else refused. A picture that is all black is refused ("화면이 검게 나와 볼 수 없습니다",
protected video, exclusive full screen). Pictures go only into the `screen_action_result`: never
saved, never logged.
- **List**: monitors `{number, width, height, main}` and visible windows `{program, title,
minimised}`, front first, at most 30, without excluded programs and the app's own windows.
- **Activity**: each capture is `screen.view` with the target (`전체 화면`, `모니터 <n>`, `창`; never
a title), who asked and whether it worked ("Activity").
- Logs carry the action, target kind, size and time taken, never a title or a picture.
## Personal notes (`app/src/notes.rs`)
`docs/v2/proactive.md` (제안 4). Each person's standing instructions for how Jarvis answers them,
@@ -184,7 +220,7 @@ editable on the 사람별 지침 screen.
`docs/v2/proactive.md` (제안 5). What Jarvis did outside its answers, for the 한 일 screen: each chat
channel action the brain's tool asked for (`chat.send`, `chat.edit`, `chat.react`, `chat.delete`,
`chat.pin`, `chat.unpin`), each schedule delivery (`schedule.remind`, `schedule.task`) and each
news notification (`collect.notify`), done or failed. PC control actions join it with M7. Ordinary answers are not recorded.
news notification (`collect.notify`), each screen capture (`screen.view`, "Screen viewing"), done or failed. The other PC control actions join it with M7. Ordinary answers are not recorded.
- One JSON line per action in `activity/<YYYY-MM-DD>.jsonl` (the folder is made at the first one):
`{at, kind, target, by, ok, note}`. `target` is the channel (and message) id or `예약 <id>`; `by`
@@ -530,14 +566,15 @@ The brain answers with one model, Qwen3.6-35B-A3B UD-Q4_K_XL through Strata (M0)
start it first, so nothing answers before the brain can, and stop it last.
- **Strata from the data folder** (the default): `strata launch --family qwen36 --model ud-q4_k_xl
--data-dir models/llm --context <llm.context> --vision off --parallel <n> --access local --port
--data-dir models/llm --context <llm.context> --vision <off|gpu|cpu> --parallel <n> --access local --port
<llm.port> --no-api-key --restart --yes --no-tui`, run like an install command (the layout's
environment, `runtime/strata` as the working folder, one process group, lines to the `llm` log
tab). It downloads the model on the first start, starts the engine and returns; the app then
waits (up to 30 minutes) until `/v1/models` lists the model and takes its id. `--restart`
replaces a server left with other choices; the same choices are reused. `--parallel` is 1 (faster
on a 16 GB card, M0) and 2 when a chat channel answers every message, so its answers do not wait
behind voice answers. Pictures are off until something in v2 sends one. Stopping the services
behind voice answers. Pictures (`--vision`) are off unless screen viewing is available ("Screen viewing"): then `gpu`
in the normal stage and `cpu` in the game stage (M0). Stopping the services
runs `strata launch --stop` (ended after 2 minutes, with everything it started) and then checks
that the port refuses connections; so does a failed or cancelled start, so nothing half-started
keeps graphics memory (each check waits up to 10 s to connect: Windows refuses a closed local
@@ -559,7 +596,8 @@ start it first, so nothing answers before the brain can, and stop it last.
left. What the model holds without experts is measured once per context, number of requests at
once and card (its NVIDIA UUID): the first-run self-check, a start with `expert-vram=0`, one
short answer, then the card's memory again and the staging from the engine log; kept in
`config/llm-probe.json` under `<context>/<parallel>`, and the real start then replaces that
`config/llm-probe.json` under `<context>/<parallel>` (`<context>/<parallel>/vision` with pictures
on the card), and the real start then replaces that
server. A measurement whose answer or second reading fails (or shows nothing more held) is not
kept: the model starts with `expert-vram=0` and the next start measures again. Before the card
is read, `strata launch --stop` always runs and the start's port must refuse connections, so a
@@ -821,7 +859,15 @@ also with the user idle past Windows' focus lock, no taskbar or Alt+Tab entry fo
left after the stop, and the uninstall leaving no device, driver package, registry key or settings
folder. Game detection (`tests/game_watch.rs`): a program started from a Steam game folder (also under
a path with non-ASCII letters) or from a Battle.net game's install folder (a test uninstall entry)
turns game mode on with its name and off once it ends. Performance stages (on every system, with a fake
turns game mode on with its name and off once it ends. Screen viewing (`tests/screen_windows.rs`,
`JARVIS_TEST_SCREEN=1`): the monitors and Notepad's window are listed, the window, each monitor and
the whole screen come back as JPEG pictures of their size, and with Notepad excluded it is neither
picked nor listed and its area is black on the whole screen; someone not registered is refused. On
every system with a fake screen: refusals before any capture, monitors by number, the whole screen
as placed with excluded areas black (also at another display scale), windows by title or program
front first (never excluded, secondary or the app's own), minimised and black pictures refused, the
detail sizes and parts; the worker protocol test carries a 4K capture's pictures to a fake brain and
back. Performance stages (on every system, with a fake
Strata and card): a game moves the LLM to the game placement (measured once at 16K and one request,
2 GiB on the game PC's numbers, half the 16 engine threads) and back after the wait, pause stops
it, a card with no room for a game pauses, the normal stage counts a program opened since, the game

View File

@@ -934,6 +934,33 @@ async fn worker_task(core: std::sync::Weak<Inner>, mut rx: mpsc::UnboundedReceiv
}
inner.worker.note_action_result(&a.id, result);
}
WorkerEvent::ScreenAction(a) => {
let inner = inner.clone();
tokio::spawn(async move {
let started = std::time::Instant::now();
let settings = inner.settings.clone();
let req = a.clone();
let result = tokio::task::spawn_blocking(move || match crate::screen::system_screen() {
Some(screen) => crate::screen::act(&settings, screen.as_ref(), &req),
None => Err("이 PC에서는 화면을 볼 수 없습니다".into()),
}).await.unwrap_or_else(|_| Err("화면을 캡처하지 못했습니다".into()));
// The kind of target, size and time only: never a title or a picture.
let target = crate::screen::target_of(&a).map(|(t, _)| crate::screen::activity_target(&t))
.unwrap_or_else(|_| "?".into());
match &result {
Ok(v) => debug_log!("conversation", "screen {} {target} {}x{} parts={} ms={}", a.action,
v["item"]["width"], v["item"]["height"], v["item"]["parts"], started.elapsed().as_millis()),
Err(e) => debug_log!("conversation", "screen {} {target} refused: {e}", a.action),
}
if a.action == "capture" {
if let Some(activity) = inner.activity.get() {
let by = a.user.parse().ok().map(|u| format!("{u} {}", name_of(&inner, u)));
activity.record("screen.view", &target, by, result.is_ok(), result.as_ref().err().cloned());
}
}
inner.worker.screen_action_result(&a.id, result);
});
}
WorkerEvent::ScheduleAction(a) => {
let result = match inner.schedules.get() {
Some(store) => {

View File

@@ -5,7 +5,7 @@
use serde_json::{json, Map, Value};
use crate::secrets::{Secrets, SecretsError};
use crate::settings::{FeatureSettings, McpServerSettings};
use crate::settings::{FeatureSettings, McpServerSettings, Settings};
/// The worker's environment variable with the MCP servers' environment values
/// (`voice_worker/brain.py`, `MCP_ENV_VAR`).
@@ -22,8 +22,17 @@ pub const FEATURE_TOOLS: &[(&str, &[&str])] = &[
("collect", &["collectedNews"]),
];
/// The built-in tools of the features that are off.
pub fn disabled_tools(f: &FeatureSettings) -> Vec<&'static str> {
/// The built-in tools of the features that are off, and `screenView` unless screen viewing is
/// available (admin.spec.md, "Screen viewing").
pub fn disabled_tools(s: &Settings) -> Vec<&'static str> {
let mut off = feature_tools_off(&s.features);
if !crate::screen::available(s) {
off.push("screenView");
}
off
}
fn feature_tools_off(f: &FeatureSettings) -> Vec<&'static str> {
let on = |key: &str| match key {
"weather" => f.weather,
"webSearch" => f.web_search,
@@ -108,10 +117,18 @@ mod tests {
#[test]
fn features_that_are_off_turn_their_tools_off() {
let all_on = FeatureSettings { weather: true, web_search: true, meals: true, collect: true, browser: true, browser_share: true, schedule: true, notes: true };
assert!(disabled_tools(&all_on).is_empty());
let off = disabled_tools(&FeatureSettings { weather: false, web_search: true, meals: false, collect: false, browser: false, browser_share: false, schedule: false, notes: false });
assert_eq!(off, vec!["getWeather", "logMeal", "fetchMeals", "deleteMeal", "botBrowser", "schedule", "personalNotes", "collectedNews"]);
let mut s = Settings::default();
s.pc_control.allowed_users = vec![1];
let screen = if cfg!(windows) { vec![] } else { vec!["screenView"] };
assert_eq!(disabled_tools(&s), screen, "all on by default");
s.features = FeatureSettings { weather: false, web_search: true, meals: false, collect: false, browser: false, browser_share: false, schedule: false, notes: false, screen: true };
let mut want = vec!["getWeather", "logMeal", "fetchMeals", "deleteMeal", "botBrowser", "schedule", "personalNotes", "collectedNews"];
want.extend(&screen);
assert_eq!(disabled_tools(&s), want);
// Screen viewing needs someone registered for PC control (and Windows).
assert!(disabled_tools(&Settings::default()).contains(&"screenView"));
s.features.screen = false;
assert!(disabled_tools(&s).contains(&"screenView"));
}
#[test]

View File

@@ -120,12 +120,12 @@ pub struct Detected {
pub reason: Reason,
}
fn file_name(path: &str) -> String {
pub(crate) fn file_name(path: &str) -> String {
path.rsplit(['\\', '/']).next().unwrap_or(path).to_lowercase()
}
/// Whether `name` (a program's file name, lower case) is in `list`; list entries may leave out ".exe".
fn listed(name: &str, list: &[String]) -> bool {
pub(crate) fn listed(name: &str, list: &[String]) -> bool {
list.iter().map(|e| e.trim().to_lowercase()).any(|e| e == name || format!("{e}.exe") == name)
}
@@ -313,7 +313,7 @@ mod probe {
}
#[cfg(windows)]
mod probe {
pub(crate) mod probe {
use super::{battle_net_game, Foreground, InstalledGame, Snapshot};
use windows_sys::Win32::System::Registry::{
RegCloseKey, RegEnumKeyExW, RegGetValueW, RegOpenKeyExW, HKEY, HKEY_LOCAL_MACHINE, KEY_READ, KEY_WOW64_32KEY,
@@ -491,7 +491,7 @@ mod probe {
}
/// Process `pid`'s program path.
fn image_path(pid: u32) -> Option<String> {
pub(crate) fn image_path(pid: u32) -> Option<String> {
// SAFETY: a query-only handle, closed at once; the buffer is sized as passed.
unsafe {
let h = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid);

View File

@@ -30,6 +30,7 @@ pub mod placement;
pub mod policy;
pub mod receive;
pub mod schedule;
pub mod screen;
pub mod secrets;
pub mod services;
pub mod settings;

764
app/src/screen.rs Normal file
View File

@@ -0,0 +1,764 @@
//! Screen viewing for the brain's `screenView` tool (admin.spec.md, "Screen viewing";
//! `docs/v2/pc-features.md`, "화면 보기"): who may see, which monitor or window, the pictures it
//! gets (scaled, excluded programs painted black, JPEG). The capture itself is Windows Graphics
//! Capture (`win`); everything else is plain code the tests drive with a fake screen.
use base64::Engine as _;
use serde_json::{json, Value};
use crate::permissions::{decide, Action, Decision, Refusal};
use crate::settings::{Level, Settings};
use crate::worker_client::ScreenAction;
/// Long side of a `fast` and a `normal` picture, and of each `detailed` part.
pub const FAST_PX: u32 = 768;
pub const NORMAL_PX: u32 = 1280;
/// A picture whose long side is at most this stays one `detailed` part at full size.
pub const ONE_PART_MAX_PX: u32 = 1600;
pub const JPEG_QUALITY: u8 = 80;
/// Pictures per answer (base64 grows them by a third; the worker takes 4 MiB messages).
pub const MAX_BYTES: usize = 3 * 1024 * 1024;
/// Windows in the list.
pub const MAX_WINDOWS: usize = 30;
/// Whether the brain gets the tool and the LLM reads pictures: Windows, the feature on, someone
/// registered for PC control and viewing not turned off.
pub fn available(s: &Settings) -> bool {
cfg!(windows) && s.features.screen && !s.pc_control.allowed_users.is_empty() && s.pc_control.view != Level::Never
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Rect {
pub x: i32,
pub y: i32,
pub w: i32,
pub h: i32,
}
#[derive(Debug, Clone, PartialEq)]
pub struct Monitor {
/// As Windows numbers it (`\\.\DISPLAY<n>`).
pub number: u32,
pub rect: Rect,
pub main: bool,
/// The opaque handle the capture takes.
pub handle: isize,
}
/// A visible top-level window, front first.
#[derive(Debug, Clone, PartialEq)]
pub struct Window {
pub handle: isize,
/// Program file name, lower case (`chrome.exe`).
pub program: String,
pub title: String,
pub rect: Rect,
pub minimised: bool,
/// Owned windows and tool windows (pop-ups, palettes): never listed or picked, but painted
/// black when their program is excluded.
pub secondary: bool,
/// This app's own windows.
pub own: bool,
}
/// A captured picture: top-down BGRA rows.
#[derive(Debug, Clone, PartialEq)]
pub struct Image {
pub w: u32,
pub h: u32,
pub bgra: Vec<u8>,
}
impl Image {
pub fn new(w: u32, h: u32) -> Self {
Image { w, h, bgra: vec![0; (w * h * 4) as usize] }
}
}
/// What the screen offers; Windows in `win`, a fake in the tests.
pub trait Screen {
/// The monitors without the bot browser's virtual one.
fn monitors(&self) -> Result<Vec<Monitor>, String>;
fn windows(&self) -> Result<Vec<Window>, String>;
fn capture_monitor(&self, m: &Monitor) -> Result<Image, String>;
fn capture_window(&self, w: &Window) -> Result<Image, String>;
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Detail {
Fast,
Normal,
Detailed,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Target {
Screen,
Monitor(u32),
Window(String),
}
/// The activity log's target: never a title.
pub fn activity_target(t: &Target) -> String {
match t {
Target::Screen => "전체 화면".into(),
Target::Monitor(n) => format!("모니터 {n}"),
Target::Window(_) => "창".into(),
}
}
/// The request in a `capture` action.
pub fn target_of(a: &ScreenAction) -> Result<(Target, Detail), String> {
let detail = match a.detail.as_deref() {
Some("fast") => Detail::Fast,
Some("detailed") => Detail::Detailed,
_ => Detail::Normal,
};
let target = match a.target.as_deref() {
Some("monitor") => Target::Monitor(a.monitor.filter(|n| *n >= 1).ok_or("모니터 번호가 필요합니다")?),
Some("window") => {
let w = a.window.as_deref().map(str::trim).filter(|w| !w.is_empty()).ok_or("창 이름이 필요합니다")?;
Target::Window(w.chars().take(200).collect())
}
_ => Target::Screen,
};
Ok((target, detail))
}
/// Why the asker may not see the screen, or `None` when they may.
pub fn refusal(s: &Settings, user: &str) -> Option<String> {
let Ok(id) = user.parse::<u64>() else { return Some(not_registered()) };
match decide(&s.pc_control, id, Action::View) {
Decision::Allow => None,
Decision::Refuse(Refusal::NotAllowed) => Some(not_registered()),
// `ask` needs the next-message consent the PC control tool brings (M7, next step).
Decision::Ask => Some("화면 보기가 '먼저 묻기'로 되어 있어 지금은 볼 수 없습니다(PC 제어 화면에서 '자동'으로 바꾸면 됩니다)".into()),
Decision::Refuse(_) => Some("화면 보기가 꺼져 있습니다".into()),
}
}
fn not_registered() -> String {
"PC 제어 화면에 등록된 사람만 화면을 볼 수 있습니다".into()
}
fn excluded(s: &Settings, program: &str) -> bool {
crate::game::listed(program, &s.pc_control.screen_exclude)
}
/// The windows the list shows and a window request may pick: not secondary, not this app's, not
/// excluded; front first.
fn listable<'a>(s: &'a Settings, all: &'a [Window]) -> impl Iterator<Item = &'a Window> + 'a {
all.iter().filter(move |w| !w.secondary && !w.own && !w.title.trim().is_empty() && !excluded(s, &w.program))
}
/// The window nearest the front whose title or program name contains `text` (any case).
pub fn pick_window<'a>(s: &'a Settings, all: &'a [Window], text: &str) -> Option<&'a Window> {
let t = text.to_lowercase();
let stem = |p: &str| p.strip_suffix(".exe").unwrap_or(p).to_owned();
listable(s, all).find(|w| w.title.to_lowercase().contains(&t) || stem(&w.program).contains(&t))
}
/// Whether nothing shows (protected video, exclusive full screen): every pixel near black.
pub fn all_black(img: &Image) -> bool {
img.bgra.chunks_exact(4).all(|p| p[0] <= 8 && p[1] <= 8 && p[2] <= 8)
}
/// Paint `rects` (screen coordinates) black on a picture of `area`; the picture may have another
/// size than `area` (display scaling), so the rectangles are scaled to it.
pub fn paint_black(img: &mut Image, area: Rect, rects: &[Rect]) {
if area.w <= 0 || area.h <= 0 {
return;
}
let (sx, sy) = (img.w as f64 / area.w as f64, img.h as f64 / area.h as f64);
for r in rects {
let x0 = (((r.x - area.x) as f64 * sx).floor().max(0.0) as u32).min(img.w);
let y0 = (((r.y - area.y) as f64 * sy).floor().max(0.0) as u32).min(img.h);
let x1 = (((r.x + r.w - area.x) as f64 * sx).ceil().max(0.0) as u32).min(img.w);
let y1 = (((r.y + r.h - area.y) as f64 * sy).ceil().max(0.0) as u32).min(img.h);
for y in y0..y1 {
let row = (y * img.w) as usize * 4;
img.bgra[row + x0 as usize * 4..row + x1 as usize * 4].fill(0);
}
}
}
/// `img` scaled down (area average) so its long side is at most `max`; never up.
pub fn scale(img: &Image, max: u32) -> Image {
let long = img.w.max(img.h);
if long <= max || img.w == 0 || img.h == 0 {
return img.clone();
}
let (w, h) = (((img.w as u64 * max as u64) / long as u64).max(1) as u32, ((img.h as u64 * max as u64) / long as u64).max(1) as u32);
let mut out = Image::new(w, h);
let (fx, fy) = (img.w as f64 / w as f64, img.h as f64 / h as f64);
for oy in 0..h {
let (y0, y1) = ((oy as f64 * fy) as u32, (((oy + 1) as f64 * fy).ceil() as u32).min(img.h));
for ox in 0..w {
let (x0, x1) = ((ox as f64 * fx) as u32, (((ox + 1) as f64 * fx).ceil() as u32).min(img.w));
let mut sum = [0u32; 3];
for y in y0..y1 {
let row = (y * img.w) as usize * 4;
for x in x0..x1 {
let p = &img.bgra[row + x as usize * 4..];
sum[0] += p[0] as u32;
sum[1] += p[1] as u32;
sum[2] += p[2] as u32;
}
}
let n = ((y1 - y0) * (x1 - x0)).max(1);
let o = ((oy * w + ox) * 4) as usize;
out.bgra[o..o + 4].copy_from_slice(&[(sum[0] / n) as u8, (sum[1] / n) as u8, (sum[2] / n) as u8, 255]);
}
}
out
}
fn crop(img: &Image, x: u32, y: u32, w: u32, h: u32) -> Image {
let mut out = Image::new(w, h);
for row in 0..h {
let src = (((y + row) * img.w + x) * 4) as usize;
let dst = (row * w * 4) as usize;
out.bgra[dst..dst + (w * 4) as usize].copy_from_slice(&img.bgra[src..src + (w * 4) as usize]);
}
out
}
/// The pictures of a request: one scaled picture, or for `detailed` four parts (left to right,
/// top to bottom) of at most `NORMAL_PX` each.
pub fn parts(img: &Image, detail: Detail) -> Vec<Image> {
match detail {
Detail::Fast => vec![scale(img, FAST_PX)],
Detail::Normal => vec![scale(img, NORMAL_PX)],
Detail::Detailed if img.w.max(img.h) <= ONE_PART_MAX_PX => vec![img.clone()],
Detail::Detailed => {
let (hw, hh) = (img.w / 2, img.h / 2);
[(0, 0, hw, hh), (hw, 0, img.w - hw, hh), (0, hh, hw, img.h - hh), (hw, hh, img.w - hw, img.h - hh)]
.into_iter().map(|(x, y, w, h)| scale(&crop(img, x, y, w, h), NORMAL_PX)).collect()
}
}
}
pub fn jpeg(img: &Image) -> Result<Vec<u8>, String> {
let mut out = Vec::new();
jpeg_encoder::Encoder::new(&mut out, JPEG_QUALITY)
.encode(&img.bgra, img.w as u16, img.h as u16, jpeg_encoder::ColorType::Bgra)
.map_err(|e| format!("그림을 만들지 못했습니다: {e}"))?;
Ok(out)
}
fn bounds(ms: &[Monitor]) -> Rect {
let x0 = ms.iter().map(|m| m.rect.x).min().unwrap_or(0);
let y0 = ms.iter().map(|m| m.rect.y).min().unwrap_or(0);
let x1 = ms.iter().map(|m| m.rect.x + m.rect.w).max().unwrap_or(0);
let y1 = ms.iter().map(|m| m.rect.y + m.rect.h).max().unwrap_or(0);
Rect { x: x0, y: y0, w: x1 - x0, h: y1 - y0 }
}
/// One picture of `ms` as they are placed; each monitor's capture is fitted to its place.
fn whole(screen: &dyn Screen, ms: &[Monitor], excluded_rects: &[Rect]) -> Result<(Image, bool), String> {
let b = bounds(ms);
let mut out = Image::new(b.w.max(1) as u32, b.h.max(1) as u32);
let mut black = true;
for m in ms {
let shot = screen.capture_monitor(m)?;
black &= all_black(&shot);
let mut fit = if shot.w as i32 == m.rect.w && shot.h as i32 == m.rect.h { shot } else { stretch(&shot, m.rect) };
paint_black(&mut fit, m.rect, excluded_rects);
let (ox, oy) = ((m.rect.x - b.x) as u32, (m.rect.y - b.y) as u32);
for row in 0..fit.h.min(out.h - oy) {
let w = fit.w.min(out.w - ox) as usize * 4;
let src = (row * fit.w * 4) as usize;
let dst = (((oy + row) * out.w + ox) * 4) as usize;
out.bgra[dst..dst + w].copy_from_slice(&fit.bgra[src..src + w]);
}
}
Ok((out, black))
}
/// `img` resized to exactly `r`'s size (nearest pixel): monitors whose capture differs from their
/// placed size (display scaling) still fill their place.
fn stretch(img: &Image, r: Rect) -> Image {
let (w, h) = (r.w.max(1) as u32, r.h.max(1) as u32);
let mut out = Image::new(w, h);
for y in 0..h {
let sy = (y as u64 * img.h as u64 / h as u64) as u32;
for x in 0..w {
let sx = (x as u64 * img.w as u64 / w as u64) as u32;
let s = ((sy * img.w + sx) * 4) as usize;
let d = ((y * w + x) * 4) as usize;
out.bgra[d..d + 4].copy_from_slice(&img.bgra[s..s + 4]);
}
}
out
}
/// Answer one `screen_action` for its asker: `{item, images}` or why not.
pub fn act(s: &Settings, screen: &dyn Screen, a: &ScreenAction) -> Result<Value, String> {
if let Some(why) = refusal(s, &a.user) {
return Err(why);
}
match a.action.as_str() {
"list" => {
let monitors: Vec<Value> = screen.monitors()?.iter().map(|m| json!({"number": m.number,
"width": m.rect.w, "height": m.rect.h, "main": m.main})).collect();
let all = screen.windows()?;
let windows: Vec<Value> = listable(s, &all).take(MAX_WINDOWS).map(|w| json!({
"program": w.program.strip_suffix(".exe").unwrap_or(&w.program), "title": w.title,
"minimised": w.minimised})).collect();
Ok(json!({"item": {"monitors": monitors, "windows": windows}}))
}
"capture" => {
let (target, detail) = target_of(a)?;
let (img, black, what) = match &target {
Target::Window(text) => {
let all = screen.windows()?;
let w = pick_window(s, &all, text).ok_or("그런 창을 찾지 못했습니다")?;
if w.minimised {
return Err("그 창은 최소화되어 있어 볼 수 없습니다".into());
}
let img = screen.capture_window(w)?;
let black = all_black(&img);
(img, black, format!("창 {}", w.program.strip_suffix(".exe").unwrap_or(&w.program)))
}
_ => {
let ms = screen.monitors()?;
let chosen: Vec<Monitor> = match target {
Target::Monitor(n) => vec![ms.iter().find(|m| m.number == n).cloned()
.ok_or_else(|| format!("{n}번 모니터가 없습니다 (있는 모니터: {})",
ms.iter().map(|m| m.number.to_string()).collect::<Vec<_>>().join(", ")))?],
_ => ms,
};
if chosen.is_empty() {
return Err("모니터를 찾지 못했습니다".into());
}
let hidden: Vec<Rect> = screen.windows()?.iter()
.filter(|w| !w.minimised && excluded(s, &w.program)).map(|w| w.rect).collect();
let (img, black) = whole(screen, &chosen, &hidden)?;
(img, black, activity_target(&target))
}
};
if black {
return Err("화면이 검게 나와 볼 수 없습니다 (보호된 영상이나 전체 화면 게임일 수 있습니다)".into());
}
let pics = parts(&img, detail);
let mut images = Vec::with_capacity(pics.len());
let mut total = 0;
for p in &pics {
let j = jpeg(p)?;
total += j.len();
images.push(base64::engine::general_purpose::STANDARD.encode(j));
}
if total > MAX_BYTES {
return Err("그림이 너무 커서 보낼 수 없습니다 (화질을 낮춰 다시 보세요)".into());
}
Ok(json!({"item": {"what": what, "width": img.w, "height": img.h, "parts": images.len()}, "images": images}))
}
other => Err(format!("모르는 화면 동작입니다: {}", other.chars().take(20).collect::<String>())),
}
}
/// The real screen (Windows), or none elsewhere.
pub fn system_screen() -> Option<Box<dyn Screen + Send>> {
#[cfg(windows)]
return Some(Box::new(win::System));
#[cfg(not(windows))]
None
}
#[cfg(windows)]
mod win {
use super::{Image, Monitor, Rect, Screen, Window};
use std::time::{Duration, Instant};
use windows::core::Interface;
use windows::Graphics::Capture::{Direct3D11CaptureFramePool, GraphicsCaptureItem};
use windows::Graphics::DirectX::Direct3D11::IDirect3DDevice;
use windows::Graphics::DirectX::DirectXPixelFormat;
use windows::Win32::Foundation::{HMODULE, HWND};
use windows::Win32::Graphics::Direct3D::{D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_WARP};
use windows::Win32::Graphics::Direct3D11::*;
use windows::Win32::Graphics::Dxgi::IDXGIDevice;
use windows::Win32::Graphics::Gdi::HMONITOR;
use windows::Win32::System::WinRT::Direct3D11::{CreateDirect3D11DeviceFromDXGIDevice, IDirect3DDxgiInterfaceAccess};
use windows::Win32::System::WinRT::Graphics::Capture::IGraphicsCaptureItemInterop;
use windows::Win32::System::WinRT::{RoInitialize, RO_INIT_MULTITHREADED};
use windows_sys::core::BOOL;
use windows_sys::Win32::Foundation::{LPARAM, RECT};
use windows_sys::Win32::Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_CLOAKED};
use windows_sys::Win32::Graphics::Gdi::{EnumDisplayMonitors, GetMonitorInfoW, HDC, MONITORINFOEXW};
use windows_sys::Win32::UI::WindowsAndMessaging::*;
/// The first frame of a capture.
const FRAME_TIMEOUT: Duration = Duration::from_secs(2);
pub struct System;
fn text(buf: &[u16]) -> String {
String::from_utf16_lossy(&buf[..buf.iter().position(|&c| c == 0).unwrap_or(buf.len())])
}
unsafe extern "system" fn add_monitor(h: windows_sys::Win32::Graphics::Gdi::HMONITOR, _: HDC, _: *mut RECT, lp: LPARAM) -> BOOL {
// SAFETY: `lp` is the Vec `monitors` passed, alive for the whole enumeration.
let out = unsafe { &mut *(lp as *mut Vec<(isize, String, Rect, bool)>) };
let mut mi: MONITORINFOEXW = unsafe { std::mem::zeroed() };
mi.monitorInfo.cbSize = std::mem::size_of::<MONITORINFOEXW>() as u32;
// SAFETY: `mi` is a MONITORINFOEXW with its size set.
if unsafe { GetMonitorInfoW(h, &mut mi as *mut MONITORINFOEXW as *mut _) } != 0 {
let r = mi.monitorInfo.rcMonitor;
out.push((h as isize, text(&mi.szDevice),
Rect { x: r.left, y: r.top, w: r.right - r.left, h: r.bottom - r.top },
mi.monitorInfo.dwFlags & MONITORINFOF_PRIMARY != 0));
}
1
}
unsafe extern "system" fn add_window(hwnd: windows_sys::Win32::Foundation::HWND, lp: LPARAM) -> BOOL {
// SAFETY: `lp` is the Vec `windows` passed, alive for the whole enumeration.
let out = unsafe { &mut *(lp as *mut Vec<windows_sys::Win32::Foundation::HWND>) };
out.push(hwnd);
1
}
fn window(hwnd: windows_sys::Win32::Foundation::HWND, own: u32) -> Option<Window> {
// SAFETY: plain queries on a window handle from EnumWindows; a stale one only fails them.
unsafe {
if IsWindowVisible(hwnd) == 0 {
return None;
}
let mut cloaked: u32 = 0;
DwmGetWindowAttribute(hwnd, DWMWA_CLOAKED as u32, (&mut cloaked as *mut u32).cast(), 4);
if cloaked != 0 {
return None;
}
let mut buf = [0u16; 512];
let n = GetWindowTextW(hwnd, buf.as_mut_ptr(), buf.len() as i32).max(0) as usize;
let mut pid = 0;
GetWindowThreadProcessId(hwnd, &mut pid);
let mut r: RECT = std::mem::zeroed();
GetWindowRect(hwnd, &mut r);
let ex = GetWindowLongPtrW(hwnd, GWL_EXSTYLE) as u32;
let program = crate::game::probe::image_path(pid).map(|p| crate::game::file_name(&p)).unwrap_or_default();
Some(Window { handle: hwnd as isize, program, title: String::from_utf16_lossy(&buf[..n]),
rect: Rect { x: r.left, y: r.top, w: r.right - r.left, h: r.bottom - r.top },
minimised: IsIconic(hwnd) != 0,
secondary: !GetWindow(hwnd, GW_OWNER).is_null() || ex & WS_EX_TOOLWINDOW != 0,
own: pid == own })
}
}
fn device() -> windows::core::Result<(ID3D11Device, ID3D11DeviceContext, IDirect3DDevice)> {
let make = |kind: D3D_DRIVER_TYPE| -> windows::core::Result<(ID3D11Device, ID3D11DeviceContext)> {
let (mut d, mut c) = (None, None);
// SAFETY: out pointers to locals; no adapter, no software module.
unsafe { D3D11CreateDevice(None, kind, HMODULE::default(), D3D11_CREATE_DEVICE_BGRA_SUPPORT, None,
D3D11_SDK_VERSION, Some(&mut d), None, Some(&mut c))? };
Ok((d.unwrap(), c.unwrap()))
};
// Without a graphics card (a virtual machine) WARP still captures.
let (d, c) = make(D3D_DRIVER_TYPE_HARDWARE).or_else(|_| make(D3D_DRIVER_TYPE_WARP))?;
let dxgi: IDXGIDevice = d.cast()?;
// SAFETY: a valid DXGI device.
let winrt: IDirect3DDevice = unsafe { CreateDirect3D11DeviceFromDXGIDevice(&dxgi)? }.cast()?;
Ok((d, c, winrt))
}
fn capture(item: GraphicsCaptureItem) -> Result<Image, String> {
let e = |e: windows::core::Error| format!("화면을 캡처하지 못했습니다 ({:#x})", e.code().0);
// SAFETY: a second call on a thread already initialised only reports it.
let _ = unsafe { RoInitialize(RO_INIT_MULTITHREADED) };
let (dev, ctx, winrt) = device().map_err(e)?;
let size = item.Size().map_err(e)?;
let pool = Direct3D11CaptureFramePool::CreateFreeThreaded(&winrt, DirectXPixelFormat::B8G8R8A8UIntNormalized, 1, size)
.map_err(e)?;
let session = pool.CreateCaptureSession(&item).map_err(e)?;
let _ = session.SetIsCursorCaptureEnabled(false);
// Windows 11 lets the app leave out the yellow border; older Windows refuses, which is fine.
let _ = session.SetIsBorderRequired(false);
session.StartCapture().map_err(e)?;
let started = Instant::now();
let frame = loop {
if let Ok(f) = pool.TryGetNextFrame() {
break f;
}
if started.elapsed() > FRAME_TIMEOUT {
let _ = session.Close();
let _ = pool.Close();
return Err("화면이 제때 오지 않았습니다".into());
}
std::thread::sleep(Duration::from_millis(5));
};
let result = (|| -> windows::core::Result<Image> {
let content = frame.ContentSize()?;
let access: IDirect3DDxgiInterfaceAccess = frame.Surface()?.cast()?;
// SAFETY: the frame's surface is a D3D11 texture.
let tex: ID3D11Texture2D = unsafe { access.GetInterface()? };
let mut desc = D3D11_TEXTURE2D_DESC::default();
// SAFETY: `desc` is the out parameter.
unsafe { tex.GetDesc(&mut desc) };
let (w, h) = ((content.Width.max(1) as u32).min(desc.Width), (content.Height.max(1) as u32).min(desc.Height));
desc.Usage = D3D11_USAGE_STAGING;
desc.BindFlags = 0;
desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ.0 as u32;
desc.MiscFlags = 0;
let mut staging = None;
// SAFETY: a staging copy of the frame's texture, read on this thread only.
unsafe {
dev.CreateTexture2D(&desc, None, Some(&mut staging))?;
let staging = staging.unwrap();
ctx.CopyResource(&staging, &tex);
let mut map = D3D11_MAPPED_SUBRESOURCE::default();
ctx.Map(&staging, 0, D3D11_MAP_READ, 0, Some(&mut map))?;
let mut img = Image::new(w, h);
for y in 0..h as usize {
let src = std::slice::from_raw_parts((map.pData as *const u8).add(y * map.RowPitch as usize), w as usize * 4);
img.bgra[y * w as usize * 4..(y + 1) * w as usize * 4].copy_from_slice(src);
}
ctx.Unmap(&staging, 0);
Ok(img)
}
})();
let _ = frame.Close();
let _ = session.Close();
let _ = pool.Close();
result.map_err(e)
}
fn interop() -> Result<IGraphicsCaptureItemInterop, String> {
windows::core::factory::<GraphicsCaptureItem, IGraphicsCaptureItemInterop>()
.map_err(|e| format!("화면 캡처를 쓸 수 없습니다 ({:#x})", e.code().0))
}
impl Screen for System {
fn monitors(&self) -> Result<Vec<Monitor>, String> {
let mut found: Vec<(isize, String, Rect, bool)> = vec![];
// SAFETY: the callback only pushes into `found`, which outlives the call.
unsafe { EnumDisplayMonitors(std::ptr::null_mut(), std::ptr::null(), Some(add_monitor), &mut found as *mut _ as LPARAM) };
let bot = crate::vdisplay::virtual_display_name();
Ok(found.into_iter().filter(|(_, name, _, _)| Some(name) != bot.as_ref()).map(|(h, name, rect, main)| Monitor {
number: name.trim_start_matches(r"\\.\DISPLAY").parse().unwrap_or(0), rect, main, handle: h }).collect())
}
fn windows(&self) -> Result<Vec<Window>, String> {
let mut hwnds = vec![];
// SAFETY: the callback only pushes into `hwnds`, which outlives the call.
unsafe { EnumWindows(Some(add_window), &mut hwnds as *mut _ as LPARAM) };
let own = std::process::id();
Ok(hwnds.into_iter().filter_map(|h| window(h, own)).collect())
}
fn capture_monitor(&self, m: &Monitor) -> Result<Image, String> {
// SAFETY: a monitor handle from EnumDisplayMonitors.
let item = unsafe { interop()?.CreateForMonitor::<GraphicsCaptureItem>(HMONITOR(m.handle as *mut _)) }
.map_err(|e| format!("모니터를 캡처하지 못했습니다 ({:#x})", e.code().0))?;
capture(item)
}
fn capture_window(&self, w: &Window) -> Result<Image, String> {
// SAFETY: a window handle from EnumWindows; a closed window only fails the call.
let item = unsafe { interop()?.CreateForWindow::<GraphicsCaptureItem>(HWND(w.handle as *mut _)) }
.map_err(|e| format!("창을 캡처하지 못했습니다 ({:#x})", e.code().0))?;
capture(item)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::cell::RefCell;
const OWNER: &str = "123456789012345678";
fn settings() -> Settings {
let mut s = Settings::default();
s.pc_control.allowed_users = vec![OWNER.parse().unwrap()];
s
}
fn solid(w: u32, h: u32, v: u8) -> Image {
Image { w, h, bgra: [v, v, v, 255].repeat((w * h) as usize) }
}
struct Fake {
monitors: Vec<Monitor>,
windows: Vec<Window>,
shot: Image,
captured: RefCell<Vec<String>>,
}
impl Screen for Fake {
fn monitors(&self) -> Result<Vec<Monitor>, String> {
Ok(self.monitors.clone())
}
fn windows(&self) -> Result<Vec<Window>, String> {
Ok(self.windows.clone())
}
fn capture_monitor(&self, m: &Monitor) -> Result<Image, String> {
self.captured.borrow_mut().push(format!("monitor {}", m.number));
Ok(solid(m.rect.w as u32, m.rect.h as u32, 200))
}
fn capture_window(&self, w: &Window) -> Result<Image, String> {
self.captured.borrow_mut().push(format!("window {}", w.program));
Ok(self.shot.clone())
}
}
fn win(program: &str, title: &str, rect: Rect) -> Window {
Window { handle: 1, program: program.into(), title: title.into(), rect, minimised: false, secondary: false, own: false }
}
fn fake() -> Fake {
Fake {
monitors: vec![
Monitor { number: 1, rect: Rect { x: 0, y: 0, w: 400, h: 300 }, main: true, handle: 1 },
Monitor { number: 2, rect: Rect { x: 400, y: 0, w: 200, h: 100 }, main: false, handle: 2 },
],
windows: vec![
win("keepassxc.exe", "Passwords", Rect { x: 10, y: 10, w: 100, h: 50 }),
win("chrome.exe", "메일 - Chrome", Rect { x: 0, y: 0, w: 300, h: 200 }),
Window { secondary: true, ..win("chrome.exe", "pop-up", Rect { x: 0, y: 0, w: 10, h: 10 }) },
win("notepad.exe", "메모", Rect { x: 0, y: 0, w: 50, h: 50 }),
Window { own: true, ..win("jarvis.exe", "Jarvis 관리", Rect { x: 0, y: 0, w: 50, h: 50 }) },
],
shot: solid(800, 600, 120),
captured: RefCell::new(vec![]),
}
}
fn action(action: &str, user: &str, extra: Value) -> ScreenAction {
let mut v = json!({"id": "s1", "action": action, "user": user});
v.as_object_mut().unwrap().extend(extra.as_object().unwrap().clone());
serde_json::from_value(v).unwrap()
}
fn decode(v: &Value) -> Vec<Vec<u8>> {
v["images"].as_array().unwrap().iter()
.map(|i| base64::engine::general_purpose::STANDARD.decode(i.as_str().unwrap()).unwrap()).collect()
}
#[test]
fn only_registered_people_see_the_screen_and_only_when_viewing_is_automatic() {
let f = fake();
let mut s = settings();
for user in ["42", "not a number"] {
let e = act(&s, &f, &action("capture", user, json!({}))).unwrap_err();
assert!(e.contains("등록된 사람만"), "{e}");
}
s.pc_control.view = Level::Ask;
assert!(act(&s, &f, &action("list", OWNER, json!({}))).unwrap_err().contains("먼저 묻기"));
s.pc_control.view = Level::Never;
assert!(act(&s, &f, &action("list", OWNER, json!({}))).unwrap_err().contains("꺼져"));
assert!(f.captured.borrow().is_empty(), "nothing is captured for a refusal");
}
#[test]
fn the_tool_is_available_only_with_someone_registered_and_viewing_not_off() {
let mut s = settings();
assert_eq!(available(&s), cfg!(windows));
s.features.screen = false;
assert!(!available(&s));
let mut s = settings();
s.pc_control.view = Level::Never;
assert!(!available(&s));
assert!(!available(&Settings::default()), "nobody is registered by default");
}
#[test]
fn the_whole_screen_is_one_picture_with_excluded_programs_painted_black() {
let f = fake();
let s = settings();
let v = act(&s, &f, &action("capture", OWNER, json!({"target": "screen", "detail": "detailed"}))).unwrap();
assert_eq!(*f.captured.borrow(), vec!["monitor 1", "monitor 2"]);
assert_eq!((v["item"]["width"].as_u64(), v["item"]["height"].as_u64()), (Some(600), Some(300)));
assert_eq!((v["item"]["what"].as_str(), v["item"]["parts"].as_u64()), (Some("전체 화면"), Some(1)));
assert!(decode(&v).iter().all(|j| j.starts_with(&[0xFF, 0xD8])), "JPEG pictures");
// The picture before encoding: the password manager's area is black, the rest is not; the
// corner below monitor 2 that no monitor covers is black too.
let hidden = vec![f.windows[0].rect];
let (img, black) = whole(&f, &f.monitors, &hidden).unwrap();
let at = |x: u32, y: u32| img.bgra[((y * img.w + x) * 4) as usize];
assert!(!black);
assert_eq!(at(50, 30), 0, "the excluded window is painted black");
assert!(at(200, 150) == 200 && at(500, 50) == 200, "the monitors show");
assert_eq!(at(500, 250), 0, "nothing outside the monitors");
}
#[test]
fn excluded_areas_follow_a_monitor_captured_at_another_scale() {
// A picture twice the placed size (display scaling): the black area doubles with it.
let mut img = solid(800, 600, 200);
paint_black(&mut img, Rect { x: 100, y: 0, w: 400, h: 300 }, &[Rect { x: 110, y: 10, w: 50, h: 20 }]);
let at = |x: u32, y: u32| img.bgra[((y * img.w + x) * 4) as usize];
assert_eq!((at(20, 20), at(119, 59)), (0, 0));
assert_eq!((at(121, 20), at(20, 61), at(19, 20)), (200, 200, 200));
}
#[test]
fn one_monitor_by_its_number_and_a_missing_one_names_those_there_are() {
let f = fake();
let v = act(&settings(), &f, &action("capture", OWNER, json!({"target": "monitor", "monitor": 2}))).unwrap();
assert_eq!(*f.captured.borrow(), vec!["monitor 2"]);
assert_eq!((v["item"]["what"].as_str(), v["item"]["width"].as_u64()), (Some("모니터 2"), Some(200)));
let e = act(&settings(), &f, &action("capture", OWNER, json!({"target": "monitor", "monitor": 3}))).unwrap_err();
assert!(e.contains("3번") && e.contains("1, 2"), "{e}");
}
#[test]
fn a_window_is_picked_by_title_or_program_front_first_never_an_excluded_or_own_one() {
let s = settings();
let f = fake();
let v = act(&s, &f, &action("capture", OWNER, json!({"target": "window", "window": "CHROME"}))).unwrap();
assert_eq!(*f.captured.borrow(), vec!["window chrome.exe"]);
assert_eq!(v["item"]["what"], "창 chrome");
assert_eq!(pick_window(&s, &f.windows, "메모").map(|w| w.program.as_str()), Some("notepad.exe"));
for hidden in ["keepass", "passwords", "jarvis", "pop-up"] {
assert!(pick_window(&s, &f.windows, hidden).is_none(), "{hidden}");
}
let mut f = fake();
f.windows[3].minimised = true;
assert!(act(&s, &f, &action("capture", OWNER, json!({"target": "window", "window": "메모"}))).unwrap_err()
.contains("최소화"));
}
#[test]
fn a_black_picture_is_refused_and_pictures_are_scaled_to_the_detail() {
let s = settings();
let mut f = fake();
f.shot = solid(800, 600, 0);
assert!(act(&s, &f, &action("capture", OWNER, json!({"target": "window", "window": "chrome"}))).unwrap_err()
.contains("검게"));
let big = solid(3840, 2160, 90);
assert_eq!(parts(&big, Detail::Fast).iter().map(|p| (p.w, p.h)).collect::<Vec<_>>(), vec![(768, 432)]);
assert_eq!(parts(&big, Detail::Normal).iter().map(|p| (p.w, p.h)).collect::<Vec<_>>(), vec![(1280, 720)]);
let four = parts(&big, Detail::Detailed);
assert_eq!(four.len(), 4);
assert!(four.iter().all(|p| p.w.max(p.h) <= NORMAL_PX));
assert_eq!(parts(&solid(1600, 900, 90), Detail::Detailed).len(), 1);
// Small pictures are never scaled up.
assert_eq!(parts(&solid(300, 200, 90), Detail::Normal)[0].w, 300);
}
#[test]
fn the_list_shows_monitors_and_windows_without_excluded_secondary_or_own_ones() {
let v = act(&settings(), &fake(), &action("list", OWNER, json!({}))).unwrap();
assert_eq!(v["item"]["monitors"], json!([{"number": 1, "width": 400, "height": 300, "main": true},
{"number": 2, "width": 200, "height": 100, "main": false}]));
let programs: Vec<&str> = v["item"]["windows"].as_array().unwrap().iter().map(|w| w["program"].as_str().unwrap()).collect();
assert_eq!(programs, vec!["chrome", "notepad"]);
assert!(v.get("images").is_none());
}
#[test]
fn scaling_averages_the_area() {
let mut img = solid(4, 2, 0);
for y in 0..2 {
for x in 0..2 {
let o = ((y * 4 + x) * 4) as usize;
img.bgra[o..o + 3].fill(200);
}
}
let half = scale(&img, 2);
assert_eq!((half.w, half.h), (2, 1));
assert_eq!(&half.bgra[..3], &[200, 200, 200]);
assert_eq!(&half.bgra[4..7], &[0, 0, 0]);
}
}

View File

@@ -102,6 +102,8 @@ pub struct FeatureSettings {
pub schedule: bool,
/// Each person's standing instructions (admin.spec.md, "Personal notes").
pub notes: bool,
/// Screen viewing (admin.spec.md, "Screen viewing").
pub screen: bool,
}
/// Game detection (admin.spec.md, "Game detection"): on by default; the person's own games and
@@ -181,6 +183,8 @@ pub struct PcControlSettings {
pub type_text: Level,
/// Starting programs (default ask).
pub run: Level,
/// Programs never seen on the screen (file names, ".exe" may be left out).
pub screen_exclude: Vec<String>,
}
/// An MCP server the brain starts (stdio). The values of `env` are in the secrets, never here.
@@ -260,13 +264,14 @@ impl Default for Settings {
response: PolicySettings::default(),
memory: Map::new(),
llm: LlmSettings { url: None, port: 8080, context: 32768 },
features: FeatureSettings { weather: true, web_search: true, meals: true, collect: true, browser: true, browser_share: false, schedule: true, notes: true },
features: FeatureSettings { weather: true, web_search: true, meals: true, collect: true, browser: true, browser_share: false, schedule: true, notes: true, screen: true },
mcp_servers: vec![],
collect: CollectSettings { keep_days: 30, targets: vec![], notify_channel: None, digest_hour: 9 },
game: GameSettings { detect: true, games: vec![], not_games: vec![], other_room_gb: 1.0, reserve_gb: 8.0, revert_seconds: 30, pause_in_game: false },
schedule_channel: None,
pc_control: PcControlSettings { allowed_users: vec![], view: Level::Auto, click: Level::Ask,
type_text: Level::Ask, run: Level::Ask },
type_text: Level::Ask, run: Level::Ask,
screen_exclude: ["1Password", "Bitwarden", "KeePass", "KeePassXC"].map(String::from).into() },
}
}
}
@@ -597,6 +602,7 @@ pub fn parse_settings(raw: &Value) -> Result<Settings, SettingsError> {
browser_share: get_bool(f, "browserShare", "features", d.features.browser_share)?,
schedule: get_bool(f, "schedule", "features", d.features.schedule)?,
notes: get_bool(f, "notes", "features", d.features.notes)?,
screen: get_bool(f, "screen", "features", d.features.screen)?,
}
},
mcp_servers: parse_mcp_servers(raw)?,
@@ -624,6 +630,14 @@ fn parse_pc_control(raw: &Map<String, Value>, d: &PcControlSettings) -> Result<P
click: level("click", d.click)?,
type_text: level("type", d.type_text)?,
run: level("run", d.run)?,
screen_exclude: {
let v: Vec<String> = get_str_list(p, "screenExclude", "pcControl", &d.screen_exclude)?.into_iter()
.map(|e| e.trim().to_owned()).filter(|e| !e.is_empty()).collect();
if v.len() > 100 || v.iter().any(|e| e.chars().count() > 100 || e.contains(['\\', '/'])) {
return err("pcControl.screenExclude must be at most 100 program names (no folders)");
}
v
},
})
}
@@ -965,13 +979,13 @@ mod tests {
#[test]
fn features_and_mcp_servers_are_read_and_checked() {
let s = parse_settings(&json!({}));
assert_eq!(s.unwrap().features, FeatureSettings { weather: true, web_search: true, meals: true, collect: true, browser: true, browser_share: false, schedule: true, notes: true });
assert_eq!(s.unwrap().features, FeatureSettings { weather: true, web_search: true, meals: true, collect: true, browser: true, browser_share: false, schedule: true, notes: true, screen: true });
let s = parse_settings(&json!({
"features": {"weather": false, "meals": false, "browserShare": true, "schedule": false, "notes": false},
"features": {"weather": false, "meals": false, "browserShare": true, "schedule": false, "notes": false, "screen": false},
"mcpServers": [{"name": "brave", "command": "npx", "args": ["-y", "brave"], "env": ["BRAVE_API_KEY", " "]},
{"name": "time-2", "command": "uvx", "enabled": false}]
})).unwrap();
assert_eq!(s.features, FeatureSettings { weather: false, web_search: true, meals: false, collect: true, browser: true, browser_share: true, schedule: false, notes: false });
assert_eq!(s.features, FeatureSettings { weather: false, web_search: true, meals: false, collect: true, browser: true, browser_share: true, schedule: false, notes: false, screen: false });
assert_eq!(s.mcp_servers, vec![
McpServerSettings { name: "brave".into(), command: "npx".into(), args: vec!["-y".into(), "brave".into()],
env: vec!["BRAVE_API_KEY".into()], enabled: true },
@@ -981,11 +995,14 @@ mod tests {
let pc = parse_settings(&json!({})).unwrap().pc_control;
assert!(pc.allowed_users.is_empty());
assert_eq!((pc.view, pc.click, pc.type_text, pc.run), (Level::Auto, Level::Ask, Level::Ask, Level::Ask));
assert!(pc.screen_exclude.contains(&"KeePassXC".to_owned()), "password managers are never seen by default");
let pc = parse_settings(&json!({"pcControl": {"allowedUsers": ["123456789012345678"], "view": "never",
"click": "auto", "type": "never", "run": "auto"}})).unwrap().pc_control;
"click": "auto", "type": "never", "run": "auto", "screenExclude": [" bank.exe ", ""]}})).unwrap().pc_control;
assert_eq!(pc.allowed_users, vec![123456789012345678]);
assert_eq!(pc.screen_exclude, vec!["bank.exe".to_owned()]);
assert_eq!((pc.view, pc.click, pc.type_text, pc.run), (Level::Never, Level::Auto, Level::Never, Level::Auto));
for bad in [json!({"pcControl": {"click": "yes"}}), json!({"pcControl": {"allowedUsers": ["1"]}})] {
for bad in [json!({"pcControl": {"click": "yes"}}), json!({"pcControl": {"allowedUsers": ["1"]}}),
json!({"pcControl": {"screenExclude": ["C:\\bank\\bank.exe"]}}), json!({"pcControl": {"screenExclude": "x"}})] {
assert!(parse_settings(&bad).unwrap_err().0.contains("pcControl."), "{bad}");
}
// A schedule channel must be a registered one that Jarvis may send in.

View File

@@ -133,6 +133,7 @@ pub fn fields() -> Vec<Field> {
f("features.browserShare", Features, Bool, json!(false), "봇용 크롬창 화면공유 선택 표시(가상 모니터, Windows)"),
f("features.schedule", Features, Bool, json!(true), "예약 알림과 정기 작업"),
f("features.notes", Features, Bool, json!(true), "사람별 지침"),
f("features.screen", Features, Bool, json!(true), "화면 보기(Windows, PC 제어에 등록한 사람만)"),
f("mcpServers", Features, McpServers, json!([]), "MCP 서버"),
// Background collection (admin.spec.md, "Background collection"): no targets by default.
f("collect.targets", Collect, CollectTargets, json!([]), "수집 대상"),
@@ -148,6 +149,8 @@ pub fn fields() -> Vec<Field> {
f("pcControl.click", PcControl, Choice { values: LEVELS }, json!("ask"), "누르기(버튼, 메뉴, 화면의 한 점)"),
f("pcControl.type", PcControl, Choice { values: LEVELS }, json!("ask"), "글자·키 입력"),
f("pcControl.run", PcControl, Choice { values: LEVELS }, json!("ask"), "프로그램 실행"),
f("pcControl.screenExclude", PcControl, TextList { max_items: 100, nullable: false },
json!(["1Password", "Bitwarden", "KeePass", "KeePassXC"]), "화면 보기에서 뺄 프로그램(파일 이름, 은행·비밀번호 관리자 등)"),
// Game detection (admin.spec.md, "Game detection").
f("game.detect", Game, Bool, json!(true), "게임 감지"),
f("game.games", Game, TextList { max_items: 100, nullable: false }, json!([]), "게임으로 볼 프로그램(파일 이름, 예: MyGame.exe)"),

View File

@@ -54,7 +54,7 @@ pub fn launch_args(layout: &Layout, settings: &Settings, conf: &Conf, sets: &[St
let models = layout.data.join("models").join("llm").to_string_lossy().into_owned();
let mut a: Vec<String> = ["launch", "--family", FAMILY, "--model", MODEL, "--data-dir"].map(String::from).into();
a.push(models);
for (k, v) in [("--context", conf.context.to_string()), ("--vision", "off".into()),
for (k, v) in [("--context", conf.context.to_string()), ("--vision", conf.vision.into()),
("--parallel", conf.parallel.to_string()), ("--access", "local".into()),
("--port", settings.llm.port.to_string())] {
a.push(k.into());
@@ -74,15 +74,34 @@ pub fn launch_args(layout: &Layout, settings: &Settings, conf: &Conf, sets: &[St
pub struct Conf {
pub context: u32,
pub parallel: u32,
/// `--vision`: `off`, `gpu` or `cpu` (`vision`).
pub vision: &'static str,
}
pub fn conf(settings: &Settings, stage: Stage) -> Conf {
let vision = vision(stage, crate::screen::available(settings));
match stage {
Stage::Gaming => Conf { context: crate::placement::GAME_CONTEXT.min(settings.llm.context), parallel: 1 },
_ => Conf { context: settings.llm.context, parallel: parallel(settings) },
Stage::Gaming => Conf { context: crate::placement::GAME_CONTEXT.min(settings.llm.context), parallel: 1, vision },
_ => Conf { context: settings.llm.context, parallel: parallel(settings), vision },
}
}
/// Where the LLM reads pictures (`--vision`): nowhere unless screen viewing is available, on the
/// card normally and on the CPU in a game (M0's game placement).
pub fn vision(stage: Stage, screen: bool) -> &'static str {
match (screen, stage) {
(false, _) => "off",
(true, Stage::Gaming) => "cpu",
(true, _) => "gpu",
}
}
/// Where the probe of a placement is kept in `config/llm-probe.json`: pictures on the card take
/// memory of their own, so they are measured apart.
pub fn probe_key(c: &Conf) -> String {
if c.vision == "gpu" { format!("{}/{}/vision", c.context, c.parallel) } else { format!("{}/{}", c.context, c.parallel) }
}
/// The engine's CPU threads from its log ("... on 16 CPU threads ...").
fn cpu_threads(log: &str) -> Option<u32> {
log.lines().rev().find_map(|l| {
@@ -229,7 +248,7 @@ pub fn write_brain_config(data: &Path, base_url: &str, model: &str, settings: &c
"ollama_base_url": base_url,
"ollama_chat_model": model,
"db_path": data.join("memory").join("jarvis.db").to_string_lossy(),
"disabled_tools": crate::features::disabled_tools(&settings.features),
"disabled_tools": crate::features::disabled_tools(settings),
"mcps": crate::features::brain_mcps(&settings.mcp_servers),
"_config_version": 1,
});
@@ -358,7 +377,7 @@ impl Llm {
*self.baseline.lock().unwrap() = Some(now_others);
}
let baseline = *self.baseline.lock().unwrap();
let key = format!("{}/{}", c.context, c.parallel);
let key = probe_key(&c);
let file = self.probe_file();
let mut all: serde_json::Map<String, Value> = std::fs::read_to_string(&file).ok()
.and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default();
@@ -650,6 +669,22 @@ pub(crate) mod tests {
assert!(a.iter().any(|x| x == "--no-tui") && a.iter().any(|x| x == "--yes"));
}
#[test]
fn pictures_are_read_only_with_screen_viewing_and_measured_apart_on_the_card() {
assert_eq!(vision(Stage::Normal, false), "off");
assert_eq!(vision(Stage::Gaming, false), "off");
assert_eq!(vision(Stage::Normal, true), "gpu");
assert_eq!(vision(Stage::Gaming, true), "cpu");
let s = Settings::default();
let c = conf(&s, Stage::Normal);
assert_eq!(c.vision, "off", "nobody is registered for PC control by default");
let a = launch_args(&layout("llm-vision"), &s, &Conf { vision: "gpu", ..c }, &[]);
assert_eq!(a[a.iter().position(|x| x == "--vision").unwrap() + 1], "gpu");
assert_eq!(probe_key(&Conf { context: 32768, parallel: 1, vision: "off" }), "32768/1");
assert_eq!(probe_key(&Conf { context: 32768, parallel: 1, vision: "cpu" }), "32768/1");
assert_eq!(probe_key(&Conf { context: 32768, parallel: 1, vision: "gpu" }), "32768/1/vision");
}
#[test]
fn a_chat_channel_answering_everything_gets_a_second_request_slot() {
let mut s = Settings::default();
@@ -679,7 +714,7 @@ pub(crate) mod tests {
assert_eq!(v["ollama_base_url"], "http://127.0.0.1:8080/v1");
assert_eq!(v["ollama_chat_model"], "m");
assert!(Path::new(v["db_path"].as_str().unwrap()).starts_with(&l.data));
assert_eq!(v["disabled_tools"], json!(crate::features::disabled_tools(&s.features)));
assert_eq!(v["disabled_tools"], json!(crate::features::disabled_tools(&s)));
assert_eq!(v["mcps"], crate::features::brain_mcps(&s.mcp_servers));
}

View File

@@ -171,7 +171,7 @@ fn uninstall(_driver: &Path) -> Result<(), String> {
}
#[cfg(windows)]
pub use win::{attach, detach, run_elevated, settle_windows, status};
pub use win::{attach, detach, run_elevated, settle_windows, status, virtual_display_name};
#[cfg(windows)]
use win::{install, uninstall};
@@ -297,6 +297,12 @@ mod win {
current(&v.name).filter(|r| r.w > 0).ok_or_else(|| "가상 모니터가 켜지지 않았습니다".into())
}
/// The virtual monitor's display name (`\\.\DISPLAY<n>`) while it is on the desktop: screen
/// viewing leaves it out (admin.spec.md, "Screen viewing").
pub fn virtual_display_name() -> Option<String> {
adapter().filter(|d| d.attached).map(|d| text(&d.name))
}
/// Take the virtual monitor off the desktop (when it is ours and on it).
pub fn detach() {
if !ours() {

View File

@@ -164,6 +164,26 @@ pub struct NoteAction {
pub note: Option<String>,
}
/// The brain's screenView tool asks the app for the person asking (voice_worker.spec.md, "Screen
/// viewing").
#[derive(Debug, Clone, Deserialize, PartialEq)]
pub struct ScreenAction {
pub id: String,
/// `capture` or `list`.
pub action: String,
pub user: String,
/// capture: `screen`, `monitor` or `window`.
#[serde(default)]
pub target: Option<String>,
#[serde(default)]
pub monitor: Option<u32>,
#[serde(default)]
pub window: Option<String>,
/// `fast`, `normal` or `detailed`.
#[serde(default)]
pub detail: Option<String>,
}
#[derive(Debug, Clone, PartialEq)]
pub enum WorkerEvent {
Ready(HelloOk),
@@ -176,6 +196,7 @@ pub enum WorkerEvent {
ChatAction(ChatAction),
ScheduleAction(ScheduleAction),
NoteAction(NoteAction),
ScreenAction(ScreenAction),
/// The brain's answer to `reply` (empty and `failed` when it gave none).
ReplyText { id: String, text: String, failed: bool },
/// Was the utterance said to Jarvis? (a failed judgement is a no)
@@ -374,6 +395,11 @@ impl WorkerClient {
self.app_action_result("note_action_result", id, result);
}
/// The outcome of a `screen_action`: `{item, images}`, or why it failed.
pub fn screen_action_result(&self, id: &str, result: Result<Value, String>) {
self.app_action_result("screen_action_result", id, result);
}
fn app_action_result(&self, kind: &str, id: &str, result: Result<Value, String>) {
let msg = match result {
Ok(Value::Object(mut m)) => {
@@ -665,6 +691,7 @@ fn on_text(inner: &Inner, t: &str) {
"chat_action" => serde_json::from_value(v).ok().map(WorkerEvent::ChatAction),
"schedule_action" => serde_json::from_value(v).ok().map(WorkerEvent::ScheduleAction),
"note_action" => serde_json::from_value(v).ok().map(WorkerEvent::NoteAction),
"screen_action" => serde_json::from_value(v).ok().map(WorkerEvent::ScreenAction),
"llm_idle" => {
if let Some(id) = v["id"].as_u64() {
inner.llm_idle.send_if_modified(|got| if id > *got { *got = id; true } else { false });

View File

@@ -26,15 +26,20 @@ class LevelTranscriber:
class FakeBrain:
"""Answers with two sentences, says only "켜" (on) requests were said to it, ranks in reverse.
"예약" adds a reminder through the schedule backend and answers with its result; a task answers
"task: <text>"."""
"예약" adds a reminder through the schedule backend and answers with its result, "화면" captures
monitor 2 through the screen backend; a task answers "task: <text>"."""
def reply(self, user, text, history, chat=None, schedule=None, task=False, notes=None, notes_backend=None):
def reply(self, user, text, history, chat=None, schedule=None, task=False, notes=None, notes_backend=None,
screen=None):
if task:
return f"task: {text}" + (f" notes={'|'.join(notes)}" if notes else "")
if "지침" in text and notes_backend is not None:
got = notes_backend.act("add", text="반말로 해")
return f"noted {got.get('item', {}).get('id')} {got.get('error', '')}".strip()
if "화면" in text and screen is not None:
got = screen.act("capture", target="monitor", monitor=2, detail="fast")
sizes = ",".join(str(len(i)) for i in got.get("images") or [])
return f"saw {sizes} {got.get('item', {}).get('what', '')} {got.get('error', '')}".strip()
if "예약" in text and schedule is not None:
got = schedule.act("add", kind="remind", text="물 마시기", at=4_000_000_000.0, repeat="daily")
return f"scheduled {got.get('item', {}).get('id')} {got.get('error', '')}".strip()

129
app/tests/screen_windows.rs Normal file
View File

@@ -0,0 +1,129 @@
//! Screen viewing on a real Windows desktop (admin.spec.md, "Screen viewing"): the monitors and
//! windows are found, a window, each monitor and the whole screen are captured as JPEG pictures of
//! the expected size, and an excluded program is neither picked nor shown. Runs only on Windows
//! with `JARVIS_TEST_SCREEN=1` (it opens Notepad); elsewhere it skips.
use std::time::{Duration, Instant};
use base64::Engine as _;
use jarvis_app::screen;
use jarvis_app::settings::Settings;
use jarvis_app::worker_client::ScreenAction;
use serde_json::{json, Value};
fn action(v: Value) -> ScreenAction {
let mut m = json!({"id": "s1", "user": "42"});
m.as_object_mut().unwrap().extend(v.as_object().unwrap().clone());
serde_json::from_value(m).unwrap()
}
/// The pictures of an answer, decoded: (width, height, RGB pixels).
fn pictures(v: &Value) -> Vec<(u32, u32, Vec<u8>)> {
v["images"].as_array().unwrap().iter().map(|i| {
let bytes = base64::engine::general_purpose::STANDARD.decode(i.as_str().unwrap()).unwrap();
let mut d = jpeg_decoder::Decoder::new(&bytes[..]);
let px = d.decode().unwrap();
let info = d.info().unwrap();
(info.width as u32, info.height as u32, px)
}).collect()
}
/// The share of near-black pixels in `r` (screen coordinates) of a picture of `area`.
fn black_share(pic: &(u32, u32, Vec<u8>), area: screen::Rect, r: screen::Rect) -> f64 {
let (w, h, px) = pic;
let (sx, sy) = (*w as f64 / area.w as f64, *h as f64 / area.h as f64);
let (x0, x1) = (((r.x - area.x) as f64 * sx) as u32 + 2, ((r.x + r.w - area.x) as f64 * sx) as u32 - 2);
let (y0, y1) = (((r.y - area.y) as f64 * sy) as u32 + 2, ((r.y + r.h - area.y) as f64 * sy) as u32 - 2);
let (mut black, mut all) = (0, 0);
for y in y0..y1.min(*h) {
for x in x0..x1.min(*w) {
let o = ((y * w + x) * 3) as usize;
all += 1;
if px[o] < 30 && px[o + 1] < 30 && px[o + 2] < 30 {
black += 1;
}
}
}
black as f64 / all.max(1) as f64
}
#[test]
fn the_real_screen_is_seen_and_excluded_programs_are_not() {
if std::env::var("JARVIS_TEST_SCREEN").is_err() {
eprintln!("skipped: set JARVIS_TEST_SCREEN=1");
return;
}
let sys = screen::system_screen().expect("Windows has a screen");
let monitors = sys.monitors().unwrap();
println!("monitors: {:?}", monitors.iter().map(|m| (m.number, m.rect, m.main)).collect::<Vec<_>>());
assert!(!monitors.is_empty());
// Closed however the test ends; no inherited output handles, so a waiting shell is not held.
struct Closed(std::process::Child);
impl Drop for Closed {
fn drop(&mut self) {
let _ = self.0.kill();
}
}
let _notepad = Closed(std::process::Command::new("notepad.exe").stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null()).stderr(std::process::Stdio::null()).spawn().unwrap());
let until = Instant::now() + Duration::from_secs(15);
let win = loop {
if let Some(w) = sys.windows().unwrap().into_iter().find(|w| w.program == "notepad.exe" && !w.secondary) {
break w;
}
assert!(Instant::now() < until, "Notepad's window did not show");
std::thread::sleep(Duration::from_millis(200));
};
std::thread::sleep(Duration::from_secs(1));
println!("notepad: {:?} minimised={}", win.rect, win.minimised);
let mut s = Settings::default();
s.pc_control.allowed_users = vec![42];
let list = screen::act(&s, sys.as_ref(), &action(json!({"action": "list"}))).unwrap();
println!("list: {} monitors, {} windows", list["item"]["monitors"].as_array().unwrap().len(),
list["item"]["windows"].as_array().unwrap().len());
assert!(list["item"]["windows"].as_array().unwrap().iter().any(|w| w["program"] == "notepad"));
let t = Instant::now();
let v = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "window", "window": "notepad"}))).unwrap();
let pics = pictures(&v);
println!("window: {} in {} ms, picture {}x{}", v["item"], t.elapsed().as_millis(), pics[0].0, pics[0].1);
assert_eq!(v["item"]["what"], "창 notepad");
assert!(pics[0].0 > 100 && pics[0].1 > 100);
for m in &monitors {
let t = Instant::now();
let v = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "monitor", "monitor": m.number, "detail": "fast"}))).unwrap();
let pics = pictures(&v);
println!("monitor {}: {} in {} ms, picture {}x{}", m.number, v["item"], t.elapsed().as_millis(), pics[0].0, pics[0].1);
assert_eq!((v["item"]["width"].as_i64(), v["item"]["height"].as_i64()), (Some(m.rect.w as i64), Some(m.rect.h as i64)));
assert!(pics[0].0.max(pics[0].1) <= screen::FAST_PX);
}
let area = screen::Rect { x: monitors.iter().map(|m| m.rect.x).min().unwrap(), y: monitors.iter().map(|m| m.rect.y).min().unwrap(),
w: monitors.iter().map(|m| m.rect.x + m.rect.w).max().unwrap() - monitors.iter().map(|m| m.rect.x).min().unwrap(),
h: monitors.iter().map(|m| m.rect.y + m.rect.h).max().unwrap() - monitors.iter().map(|m| m.rect.y).min().unwrap() };
let t = Instant::now();
let shown = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "screen", "detail": "detailed"}))).unwrap();
let shown_pics = pictures(&shown);
println!("screen: {} in {} ms, {} part(s)", shown["item"], t.elapsed().as_millis(), shown_pics.len());
let shown_black = black_share(&shown_pics[0], area, win.rect);
println!("notepad area black when shown: {shown_black:.2}");
// Excluded: never picked or listed, and painted black on the screen.
s.pc_control.screen_exclude = vec!["notepad".into()];
let e = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "window", "window": "notepad"}))).unwrap_err();
println!("excluded window: {e}");
assert!(e.contains("찾지 못했습니다"));
let list = screen::act(&s, sys.as_ref(), &action(json!({"action": "list"}))).unwrap();
assert!(!list["item"]["windows"].as_array().unwrap().iter().any(|w| w["program"] == "notepad"));
let hidden = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "screen", "detail": "detailed"}))).unwrap();
let hidden_black = black_share(&pictures(&hidden)[0], area, win.rect);
println!("notepad area black when excluded: {hidden_black:.2}");
assert!(hidden_black > 0.99, "the excluded window is black");
assert!(shown_black < 0.5, "Notepad shows when not excluded");
// Someone not registered sees nothing.
let e = screen::act(&s, sys.as_ref(), &action(json!({"action": "list", "user": "7"}))).unwrap_err();
assert!(e.contains("등록된 사람만"));
}

View File

@@ -201,6 +201,49 @@ async fn audio_and_tts_round_trip_through_the_python_server() {
.expect("note round trip");
assert_eq!(noted, "noted 1");
assert_eq!(notes.texts_of(42), vec!["반말로 해".to_string()]);
// The screenView tool's captures come to the app, which decides for the asker and sends the
// pictures back (a large message: a 4K monitor at full detail would be about 1 MB).
struct Grey;
impl jarvis_app::screen::Screen for Grey {
fn monitors(&self) -> Result<Vec<jarvis_app::screen::Monitor>, String> {
Ok(vec![jarvis_app::screen::Monitor { number: 2, rect: jarvis_app::screen::Rect { x: 0, y: 0, w: 3840, h: 2160 },
main: true, handle: 0 }])
}
fn windows(&self) -> Result<Vec<jarvis_app::screen::Window>, String> {
Ok(vec![])
}
fn capture_monitor(&self, m: &jarvis_app::screen::Monitor) -> Result<jarvis_app::screen::Image, String> {
let (w, h) = (m.rect.w as u32, m.rect.h as u32);
Ok(jarvis_app::screen::Image { w, h, bgra: (0..w * h * 4).map(|i| (i % 251) as u8 | 0x40).collect() })
}
fn capture_window(&self, _: &jarvis_app::screen::Window) -> Result<jarvis_app::screen::Image, String> {
Err("no windows".into())
}
}
let mut screen_settings = jarvis_app::settings::Settings::default();
screen_settings.pc_control.allowed_users = vec![42];
for (rid, user, want) in [("v1", 42u64, "saw "), ("v2", 7, "등록된 사람만")] {
assert!(c.reply(rid, user, "화면 봐 줘", "voice", false));
let seen = tokio::time::timeout(Duration::from_secs(10), async {
loop {
match rx.recv().await.unwrap() {
WorkerEvent::ScreenAction(a) => {
assert_eq!((a.action.as_str(), a.target.as_deref(), a.monitor, a.detail.as_deref()),
("capture", Some("monitor"), Some(2), Some("fast")));
c.screen_action_result(&a.id, jarvis_app::screen::act(&screen_settings, &Grey, &a));
}
WorkerEvent::ReplyText { id, text, .. } if id == rid => return text,
_ => {}
}
}
})
.await
.expect("screen round trip");
assert!(seen.contains(want), "{seen}");
if user == 42 {
assert!(seen.ends_with("모니터 2") && seen.split(' ').nth(1).unwrap().parse::<usize>().unwrap() > 1000, "{seen}");
}
}
let _ = std::fs::remove_dir_all(&data);
c.close();
let _ = child.kill().await;