feat: screen viewing, Jarvis sees the whole screen, a monitor or a window for registered people

- screenView tool (screen_view.spec.md): whole screen, monitor by number, window by part of its
  title or program, or the list of monitors and windows; fast, normal or detailed pictures.
- The app captures with Windows Graphics Capture (app/src/screen.rs, admin.spec.md "Screen
  viewing"), decides for the asker (only people registered for PC control), never picks, lists
  or shows excluded programs (pcControl.screenExclude, password managers by default), refuses
  minimised windows and black pictures, and records screen.view in the activity log.
- Pictures ride on the tool message (ToolExecutionResult.images); llm.py sends them to
  OpenAI-compatible servers as image parts after the tool messages. They stay in that reply only:
  not in the tool carryover, logs or prompt dumps.
- The LLM reads pictures only when screen viewing is available (--vision gpu normally, cpu in a
  game), measured apart for the placement; otherwise screenView is a disabled tool.
- Worker: screen_action / screen_action_result, 4 MiB messages.
- Eval evals/test_screen_view_tool.py; Windows check tests/screen_windows.rs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-15 17:02:59 +09:00
parent bd7b69d4f4
commit 53bccb620d
39 changed files with 1844 additions and 68 deletions

View File

@@ -5,7 +5,7 @@
use serde_json::{json, Map, Value};
use crate::secrets::{Secrets, SecretsError};
use crate::settings::{FeatureSettings, McpServerSettings};
use crate::settings::{FeatureSettings, McpServerSettings, Settings};
/// The worker's environment variable with the MCP servers' environment values
/// (`voice_worker/brain.py`, `MCP_ENV_VAR`).
@@ -22,8 +22,17 @@ pub const FEATURE_TOOLS: &[(&str, &[&str])] = &[
("collect", &["collectedNews"]),
];
/// The built-in tools of the features that are off.
pub fn disabled_tools(f: &FeatureSettings) -> Vec<&'static str> {
/// The built-in tools of the features that are off, and `screenView` unless screen viewing is
/// available (admin.spec.md, "Screen viewing").
pub fn disabled_tools(s: &Settings) -> Vec<&'static str> {
let mut off = feature_tools_off(&s.features);
if !crate::screen::available(s) {
off.push("screenView");
}
off
}
fn feature_tools_off(f: &FeatureSettings) -> Vec<&'static str> {
let on = |key: &str| match key {
"weather" => f.weather,
"webSearch" => f.web_search,
@@ -108,10 +117,18 @@ mod tests {
#[test]
fn features_that_are_off_turn_their_tools_off() {
let all_on = FeatureSettings { weather: true, web_search: true, meals: true, collect: true, browser: true, browser_share: true, schedule: true, notes: true };
assert!(disabled_tools(&all_on).is_empty());
let off = disabled_tools(&FeatureSettings { weather: false, web_search: true, meals: false, collect: false, browser: false, browser_share: false, schedule: false, notes: false });
assert_eq!(off, vec!["getWeather", "logMeal", "fetchMeals", "deleteMeal", "botBrowser", "schedule", "personalNotes", "collectedNews"]);
let mut s = Settings::default();
s.pc_control.allowed_users = vec![1];
let screen = if cfg!(windows) { vec![] } else { vec!["screenView"] };
assert_eq!(disabled_tools(&s), screen, "all on by default");
s.features = FeatureSettings { weather: false, web_search: true, meals: false, collect: false, browser: false, browser_share: false, schedule: false, notes: false, screen: true };
let mut want = vec!["getWeather", "logMeal", "fetchMeals", "deleteMeal", "botBrowser", "schedule", "personalNotes", "collectedNews"];
want.extend(&screen);
assert_eq!(disabled_tools(&s), want);
// Screen viewing needs someone registered for PC control (and Windows).
assert!(disabled_tools(&Settings::default()).contains(&"screenView"));
s.features.screen = false;
assert!(disabled_tools(&s).contains(&"screenView"));
}
#[test]