- screenView tool (screen_view.spec.md): whole screen, monitor by number, window by part of its title or program, or the list of monitors and windows; fast, normal or detailed pictures. - The app captures with Windows Graphics Capture (app/src/screen.rs, admin.spec.md "Screen viewing"), decides for the asker (only people registered for PC control), never picks, lists or shows excluded programs (pcControl.screenExclude, password managers by default), refuses minimised windows and black pictures, and records screen.view in the activity log. - Pictures ride on the tool message (ToolExecutionResult.images); llm.py sends them to OpenAI-compatible servers as image parts after the tool messages. They stay in that reply only: not in the tool carryover, logs or prompt dumps. - The LLM reads pictures only when screen viewing is available (--vision gpu normally, cpu in a game), measured apart for the placement; otherwise screenView is a disabled tool. - Worker: screen_action / screen_action_result, 4 MiB messages. - Eval evals/test_screen_view_tool.py; Windows check tests/screen_windows.rs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
161 lines
6.8 KiB
Rust
161 lines
6.8 KiB
Rust
//! The brain's features and MCP servers (`app/src/admin.spec.md`, "Features and MCP servers"):
|
|
//! what the app writes into the brain's settings, and the MCP servers' environment values, which
|
|
//! are secrets and reach the worker only through its environment.
|
|
|
|
use serde_json::{json, Map, Value};
|
|
|
|
use crate::secrets::{Secrets, SecretsError};
|
|
use crate::settings::{FeatureSettings, McpServerSettings, Settings};
|
|
|
|
/// The worker's environment variable with the MCP servers' environment values
|
|
/// (`voice_worker/brain.py`, `MCP_ENV_VAR`).
|
|
pub const MCP_ENV_VAR: &str = "JARVIS_MCP_ENV";
|
|
|
|
/// The brain's built-in tools behind each feature switch (`jarvis/tools/registry.py`).
|
|
pub const FEATURE_TOOLS: &[(&str, &[&str])] = &[
|
|
("weather", &["getWeather"]),
|
|
("webSearch", &["webSearch", "fetchWebPage"]),
|
|
("meals", &["logMeal", "fetchMeals", "deleteMeal"]),
|
|
("browser", &["botBrowser"]),
|
|
("schedule", &["schedule"]),
|
|
("notes", &["personalNotes"]),
|
|
("collect", &["collectedNews"]),
|
|
];
|
|
|
|
/// The built-in tools of the features that are off, and `screenView` unless screen viewing is
|
|
/// available (admin.spec.md, "Screen viewing").
|
|
pub fn disabled_tools(s: &Settings) -> Vec<&'static str> {
|
|
let mut off = feature_tools_off(&s.features);
|
|
if !crate::screen::available(s) {
|
|
off.push("screenView");
|
|
}
|
|
off
|
|
}
|
|
|
|
fn feature_tools_off(f: &FeatureSettings) -> Vec<&'static str> {
|
|
let on = |key: &str| match key {
|
|
"weather" => f.weather,
|
|
"webSearch" => f.web_search,
|
|
"browser" => f.browser,
|
|
"schedule" => f.schedule,
|
|
"notes" => f.notes,
|
|
"collect" => f.collect,
|
|
_ => f.meals,
|
|
};
|
|
FEATURE_TOOLS.iter().filter(|(k, _)| !on(k)).flat_map(|(_, tools)| tools.iter().copied()).collect()
|
|
}
|
|
|
|
/// The brain's `mcps`: the enabled servers, without their environment values.
|
|
pub fn brain_mcps(servers: &[McpServerSettings]) -> Value {
|
|
Value::Object(servers.iter().filter(|s| s.enabled)
|
|
.map(|s| (s.name.clone(), json!({"transport": "stdio", "command": s.command, "args": s.args})))
|
|
.collect())
|
|
}
|
|
|
|
/// Where one environment value of a server is kept in the secrets.
|
|
pub fn secret_name(server: &str, var: &str) -> String {
|
|
format!("mcp:{server}:{var}")
|
|
}
|
|
|
|
/// `{server: {NAME: value}}` for the enabled servers, with the values that are stored.
|
|
pub fn mcp_env(secrets: &Secrets, servers: &[McpServerSettings]) -> Result<Value, SecretsError> {
|
|
let mut out = Map::new();
|
|
for s in servers.iter().filter(|s| s.enabled) {
|
|
let mut vars = Map::new();
|
|
for var in &s.env {
|
|
if let Some(v) = secrets.get(&secret_name(&s.name, var))? {
|
|
vars.insert(var.clone(), Value::String(v));
|
|
}
|
|
}
|
|
if !vars.is_empty() {
|
|
out.insert(s.name.clone(), Value::Object(vars));
|
|
}
|
|
}
|
|
Ok(Value::Object(out))
|
|
}
|
|
|
|
/// Which environment values are stored: `{server: {NAME: true|false}}` (never the values).
|
|
pub fn env_status(secrets: &Secrets, servers: &[McpServerSettings]) -> Result<Value, SecretsError> {
|
|
let names = secrets.names()?;
|
|
Ok(Value::Object(servers.iter().map(|s| {
|
|
let vars = s.env.iter().map(|v| (v.clone(), Value::Bool(names.contains(&secret_name(&s.name, v))))).collect();
|
|
(s.name.clone(), Value::Object(vars))
|
|
}).collect()))
|
|
}
|
|
|
|
/// Remove the stored values of servers and names no longer listed.
|
|
pub fn prune(secrets: &Secrets, servers: &[McpServerSettings]) -> Result<(), SecretsError> {
|
|
let keep: Vec<String> = servers.iter().flat_map(|s| s.env.iter().map(|v| secret_name(&s.name, v))).collect();
|
|
for name in secrets.names()? {
|
|
if name.starts_with("mcp:") && !keep.contains(&name) {
|
|
secrets.remove(&name)?;
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::datadir::temp_dir;
|
|
use crate::settings::parse_settings;
|
|
|
|
fn servers(v: Value) -> Vec<McpServerSettings> {
|
|
parse_settings(&json!({"mcpServers": v})).unwrap().mcp_servers
|
|
}
|
|
|
|
#[test]
|
|
fn the_feature_tools_are_the_brains_tool_names() {
|
|
let registry = std::fs::read_to_string(
|
|
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../src/jarvis/tools/registry.py")).unwrap();
|
|
for (_, tools) in FEATURE_TOOLS {
|
|
for t in *tools {
|
|
assert!(registry.contains(&format!("\"{t}\":")), "{t} is not a built-in tool");
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn features_that_are_off_turn_their_tools_off() {
|
|
let mut s = Settings::default();
|
|
s.pc_control.allowed_users = vec![1];
|
|
let screen = if cfg!(windows) { vec![] } else { vec!["screenView"] };
|
|
assert_eq!(disabled_tools(&s), screen, "all on by default");
|
|
s.features = FeatureSettings { weather: false, web_search: true, meals: false, collect: false, browser: false, browser_share: false, schedule: false, notes: false, screen: true };
|
|
let mut want = vec!["getWeather", "logMeal", "fetchMeals", "deleteMeal", "botBrowser", "schedule", "personalNotes", "collectedNews"];
|
|
want.extend(&screen);
|
|
assert_eq!(disabled_tools(&s), want);
|
|
// Screen viewing needs someone registered for PC control (and Windows).
|
|
assert!(disabled_tools(&Settings::default()).contains(&"screenView"));
|
|
s.features.screen = false;
|
|
assert!(disabled_tools(&s).contains(&"screenView"));
|
|
}
|
|
|
|
#[test]
|
|
fn only_enabled_servers_reach_the_brain_and_their_values_only_through_the_environment() {
|
|
let s = servers(json!([
|
|
{"name": "brave", "command": "npx", "args": ["-y", "brave"], "env": ["BRAVE_API_KEY", "UNSET"]},
|
|
{"name": "off", "command": "uvx", "env": ["K"], "enabled": false}]));
|
|
assert_eq!(brain_mcps(&s), json!({"brave": {"transport": "stdio", "command": "npx", "args": ["-y", "brave"]}}));
|
|
let secrets = Secrets::new(&temp_dir("mcpenv").join("secrets.dat"));
|
|
secrets.set(&secret_name("brave", "BRAVE_API_KEY"), "key-1").unwrap();
|
|
secrets.set(&secret_name("off", "K"), "key-2").unwrap();
|
|
assert_eq!(mcp_env(&secrets, &s).unwrap(), json!({"brave": {"BRAVE_API_KEY": "key-1"}}));
|
|
assert_eq!(env_status(&secrets, &s).unwrap(),
|
|
json!({"brave": {"BRAVE_API_KEY": true, "UNSET": false}, "off": {"K": true}}));
|
|
}
|
|
|
|
#[test]
|
|
fn values_of_removed_servers_and_names_are_deleted() {
|
|
let secrets = Secrets::new(&temp_dir("mcpprune").join("secrets.dat"));
|
|
secrets.set(crate::secrets::DISCORD_TOKEN, "tok").unwrap();
|
|
for (srv, var) in [("a", "K1"), ("a", "K2"), ("b", "K")] {
|
|
secrets.set(&secret_name(srv, var), "v").unwrap();
|
|
}
|
|
prune(&secrets, &servers(json!([{"name": "a", "command": "x", "env": ["K1"]}]))).unwrap();
|
|
let mut left = secrets.names().unwrap();
|
|
left.sort();
|
|
assert_eq!(left, vec![crate::secrets::DISCORD_TOKEN.to_string(), secret_name("a", "K1")]);
|
|
}
|
|
}
|