diff --git a/app/src/admin.spec.md b/app/src/admin.spec.md index f679d2e..3bde05d 100644 --- a/app/src/admin.spec.md +++ b/app/src/admin.spec.md @@ -194,6 +194,21 @@ one window; the app captures and decides who may. `window` takes the visible top-level window whose title or program name contains the text (case-insensitive; the one nearest the front when several do), also when others cover it; a minimised window is refused ("최소화되어 있어 볼 수 없습니다"). +- **Screen share** (`share`, `docs/v2/pc-features.md` "다른 사람의 디스코드 화면공유 보기"): + someone's Discord screen share, seen in the window Discord pops it out to; no account or token is + touched and Jarvis never clicks Discord. Only while Jarvis is in a voice channel ("자비스가 음성 + 채널에 있을 때만 화면공유를 볼 수 있습니다") and someone there other than Jarvis shares their + screen (`self_stream`): with a `person`, the sharer whose display name contains it or is contained + in it (letters and digits, any case), else "님은 지금 화면공유 중이 아닙니다"; with none, + anyone sharing ("지금 음성 채널에서 화면공유 중인 사람이 없습니다" when nobody does). The window + is a visible Discord window (`discord.exe`, `discordptb.exe`, `discordcanary.exe`) whose title does + not end with "Discord" (the main window's does), the one whose title contains the voice channel's + name first, else the one nearest the front; Discord on the exclusion list is never seen. A + minimised one is refused ("창을 다시 열어 주세요"). With none, a voice request says "디스코드에서 + 님 화면공유를 시청하고 창 분리 버튼을 눌러 주세요." in the channel (without a name when + several share) and looks again every second for 30 s, then refuses ("화면공유 창이 분리되지 않아 + 볼 수 없습니다"); a chat request is refused at once with that instruction. Who may see is decided + before anything is said. The worker waits 45 s for a screen action ("PC control" timing). - **Exclusions**: `pcControl.screenExclude` (program file names, ".exe" may be left out; default `1Password`, `Bitwarden`, `KeePass`, `KeePassXC`). Their windows are never captured or listed, and in a monitor or whole-screen picture their area is painted black before anything leaves the @@ -212,8 +227,8 @@ one window; the app captures and decides who may. saved, never logged. - **List**: monitors `{number, width, height, main}` and visible windows `{program, title, minimised}`, front first, at most 30, without excluded programs and the app's own windows. -- **Activity**: each capture is `screen.view` with the target (`전체 화면`, `모니터 `, `창`; never - a title), who asked and whether it worked ("Activity"). +- **Activity**: each capture is `screen.view` with the target (`전체 화면`, `모니터 `, `창`, + `화면공유`; never a title or a name), who asked and whether it worked ("Activity"). - Logs carry the action, target kind, size and time taken, never a title or a picture. ## PC control (`app/src/pc.rs`) diff --git a/app/src/discord.rs b/app/src/discord.rs index ab03589..5822b29 100644 --- a/app/src/discord.rs +++ b/app/src/discord.rs @@ -856,6 +856,68 @@ fn report_voice_state(inner: &Inner) { debug_log!("voice", "reported voice {channel} with {n} members to the worker"); } +/// One `screen_action` against the real screen. +async fn screen_act(inner: &Arc, req: crate::worker_client::ScreenAction) -> Result { + let settings = inner.settings.clone(); + tokio::task::spawn_blocking(move || match crate::screen::system_screen() { + Some(screen) => crate::screen::act(&settings, screen.as_ref(), &req), + None => Err("이 PC에서는 화면을 볼 수 없습니다".into()), + }).await.unwrap_or_else(|_| Err("화면을 캡처하지 못했습니다".into())) +} + +/// How long a voice request waits for a screen share to be popped out, and how often it looks. +const SHARE_WAIT: Duration = Duration::from_secs(30); +const SHARE_POLL: Duration = Duration::from_secs(1); + +/// Jarvis's voice channel's name and the people sharing their screen there (ID, display name); +/// None outside a voice channel. +fn share_state(inner: &Inner) -> Option<(Option, Vec<(u64, String)>)> { + let (guild, channel) = inner.session.lock().unwrap().as_ref().map(|s| (s.guild_id, s.channel_id))?; + // IDs first, then names: the cache's guild is not held while `name_of` reads it again. + let (name, users): (Option, Vec) = inner.cache.get() + .and_then(|c| c.guild(GuildId::new(guild)).map(|g| ( + g.channels.get(&ChannelId::new(channel)).map(|c| c.name.clone()), + g.voice_states.values() + .filter(|v| v.channel_id.map(|c| c.get()) == Some(channel) && v.self_stream == Some(true)) + .map(|v| v.user_id.get()) + .collect()))) + .unwrap_or_default(); + let bot = inner.bot_id.get().copied(); + Some((name, users.into_iter().filter(|u| Some(*u) != bot).map(|u| (u, name_of(inner, u))).collect())) +} + +/// Someone's Discord screen share (admin.spec.md, "Screen viewing"): only while the person asked +/// about shares their screen in Jarvis's voice channel, from the window Discord pops it out to. With +/// no such window a voice request asks for it in the channel and waits `SHARE_WAIT`. +async fn share_capture(inner: &Arc, mut req: crate::worker_client::ScreenAction) -> Result { + // Who may see comes first: nothing is said or waited for for someone who may not. + if let Some(why) = crate::screen::refusal(&inner.settings, &req.user) { + return Err(why); + } + let (channel, streaming) = share_state(inner).ok_or("자비스가 음성 채널에 있을 때만 화면공유를 볼 수 있습니다")?; + let name = crate::screen::share_streamer(&streaming, req.person.as_deref())?; + req.voice_channel = channel; + let first = screen_act(inner, req.clone()).await; + if first.as_ref().err().map(String::as_str) != Some(crate::screen::NO_SHARE_WINDOW) { + return first; + } + let prompt = crate::screen::share_prompt(name.as_deref()); + if req.via.as_deref() != Some("voice") { + return Err(format!("{} 그 뒤 다시 물어봐 주세요", prompt.trim_end_matches('.'))); + } + debug_log!("conversation", "screen share {}: no popped-out window, asking and waiting", req.id); + with_playback(inner, |p| p.say(&prompt, false)); + let until = tokio::time::Instant::now() + SHARE_WAIT; + while tokio::time::Instant::now() < until { + tokio::time::sleep(SHARE_POLL).await; + let r = screen_act(inner, req.clone()).await; + if r.as_ref().err().map(String::as_str) != Some(crate::screen::NO_SHARE_WINDOW) { + return r; + } + } + Err("화면공유 창이 분리되지 않아 볼 수 없습니다".into()) +} + /// The registered chat channels and their switches, for the brain's chat tool. fn report_chat_channels(inner: &Inner) { inner.worker.chat_channels(inner.bot_id.get().copied(), &inner.settings.chat_channels); @@ -1003,12 +1065,11 @@ async fn worker_task(core: std::sync::Weak, mut rx: mpsc::UnboundedReceiv let inner = inner.clone(); tokio::spawn(async move { let started = std::time::Instant::now(); - let settings = inner.settings.clone(); - let req = a.clone(); - let result = tokio::task::spawn_blocking(move || match crate::screen::system_screen() { - Some(screen) => crate::screen::act(&settings, screen.as_ref(), &req), - None => Err("이 PC에서는 화면을 볼 수 없습니다".into()), - }).await.unwrap_or_else(|_| Err("화면을 캡처하지 못했습니다".into())); + let result = if a.action == "capture" && a.target.as_deref() == Some("share") { + share_capture(&inner, a.clone()).await + } else { + screen_act(&inner, a.clone()).await + }; // The kind of target, size and time only: never a title or a picture. let target = crate::screen::target_of(&a).map(|(t, _)| crate::screen::activity_target(&t)) .unwrap_or_else(|_| "?".into()); diff --git a/app/src/screen.rs b/app/src/screen.rs index df9f059..5b131a3 100644 --- a/app/src/screen.rs +++ b/app/src/screen.rs @@ -96,6 +96,8 @@ pub enum Target { Screen, Monitor(u32), Window(String), + /// Someone's Discord screen share, in the window Discord pops it out to. + Share, } /// The activity log's target: never a title. @@ -104,6 +106,7 @@ pub fn activity_target(t: &Target) -> String { Target::Screen => "전체 화면".into(), Target::Monitor(n) => format!("모니터 {n}"), Target::Window(_) => "창".into(), + Target::Share => "화면공유".into(), } } @@ -120,6 +123,7 @@ pub fn target_of(a: &ScreenAction) -> Result<(Target, Detail), String> { let w = a.window.as_deref().map(str::trim).filter(|w| !w.is_empty()).ok_or("창 이름이 필요합니다")?; Target::Window(w.chars().take(200).collect()) } + Some("share") => Target::Share, _ => Target::Screen, }; Ok((target, detail)) @@ -169,6 +173,55 @@ pub fn pick_window<'a>(s: &'a Settings, all: &'a [Window], text: &str) -> Option } /// Whether nothing shows (protected video, exclusive full screen): every pixel near black. +/// Discord's programs (stable, PTB, Canary). +const DISCORD: [&str; 3] = ["discord.exe", "discordptb.exe", "discordcanary.exe"]; + +/// What a share capture gives while Discord has no popped-out screen share window: the core then +/// asks for one and waits (admin.spec.md, "Screen viewing"). +pub const NO_SHARE_WINDOW: &str = "분리된 화면공유 창이 없습니다"; + +/// The window Discord pops a screen share out to: a Discord window that is not its main one (whose +/// title ends with "Discord"). Its title is the voice channel's name, so the one naming Jarvis's +/// voice channel goes first, else the one nearest the front. +pub fn share_window<'a>(s: &'a Settings, all: &'a [Window], channel: Option<&str>) -> Option<&'a Window> { + let shares: Vec<&Window> = listable(s, all) + .filter(|w| DISCORD.contains(&w.program.as_str()) && !w.title.trim().to_lowercase().ends_with("discord")) + .collect(); + let channel = channel.map(|c| c.trim().to_lowercase()).filter(|c| !c.is_empty()); + channel.and_then(|c| shares.iter().find(|w| w.title.to_lowercase().contains(&c)).copied()) + .or_else(|| shares.first().copied()) +} + +/// Letters and digits in lower case, so a spoken name and a display name compare. +fn name_key(name: &str) -> String { + name.chars().filter(|c| c.is_alphanumeric()).flat_map(char::to_lowercase).collect() +} + +/// Whose share the popped-out window should show, from the people sharing their screen in Jarvis's +/// voice channel (ID, display name): the person asked about (their name inside what was said, or +/// what was said inside their name), or with no name anyone sharing (`None` when several do). +pub fn share_streamer(streaming: &[(u64, String)], person: Option<&str>) -> Result, String> { + let person = person.map(str::trim).filter(|p| !name_key(p).is_empty()); + let Some(person) = person else { + return match streaming { + [] => Err("지금 음성 채널에서 화면공유 중인 사람이 없습니다".into()), + [(_, name)] => Ok(Some(name.clone())), + _ => Ok(None), + }; + }; + let key = name_key(person); + streaming.iter() + .find(|(_, name)| { let n = name_key(name); !n.is_empty() && (key.contains(&n) || n.contains(&key)) }) + .map(|(_, name)| Some(name.clone())) + .ok_or_else(|| format!("{}님은 지금 화면공유 중이 아닙니다", person.chars().take(40).collect::())) +} + +/// What Jarvis says in the voice channel while it waits for the share to be popped out. +pub fn share_prompt(name: Option<&str>) -> String { + let whose = name.map(|n| format!("{n}님 화면공유를")).unwrap_or_else(|| "화면공유를".into()); + format!("디스코드에서 {whose} 시청하고 창 분리 버튼을 눌러 주세요.") +} + pub fn all_black(img: &Image) -> bool { img.bgra.chunks_exact(4).all(|p| p[0] <= 8 && p[1] <= 8 && p[2] <= 8) } @@ -330,6 +383,16 @@ pub fn act(s: &Settings, screen: &dyn Screen, a: &ScreenAction) -> Result { + let all = screen.windows()?; + let w = share_window(s, &all, a.voice_channel.as_deref()).ok_or(NO_SHARE_WINDOW)?; + if w.minimised { + return Err("화면공유 창이 최소화되어 있어 볼 수 없습니다. 창을 다시 열어 주세요".into()); + } + let img = screen.capture_window(w)?; + let black = all_black(&img); + (img, black, activity_target(&target)) + } _ => { let ms = screen.monitors()?; let chosen: Vec = match target { @@ -813,6 +876,73 @@ mod tests { assert!(v.get("images").is_none()); } + fn with_share(f: &mut Fake, popout: Window) { + f.windows.insert(1, win("discord.exe", "#일반 | 우리 서버 - Discord", Rect { x: 0, y: 0, w: 1200, h: 800 })); + f.windows.insert(0, popout); + } + + #[test] + fn a_screen_share_is_seen_in_discords_popped_out_window_never_its_main_one() { + let s = settings(); + let mut f = fake(); + let mut a = action("capture", OWNER, json!({"target": "share", "person": "철수"})); + a.voice_channel = Some("게임방".into()); + // Only the main window: nothing popped out yet. + f.windows.insert(1, win("discord.exe", "#일반 | 우리 서버 - Discord", Rect { x: 0, y: 0, w: 1200, h: 800 })); + assert_eq!(act(&s, &f, &a).unwrap_err(), NO_SHARE_WINDOW); + assert!(f.captured.borrow().is_empty()); + + let mut f = fake(); + with_share(&mut f, win("discord.exe", "게임방", Rect { x: 0, y: 0, w: 960, h: 540 })); + let v = act(&s, &f, &a).unwrap(); + assert_eq!(*f.captured.borrow(), vec!["window discord.exe"]); + assert_eq!(v["item"]["what"], "화면공유"); + assert_eq!(activity_target(&target_of(&a).unwrap().0), "화면공유"); + + // A popped-out window that is minimised is asked to be opened again. + let mut f = fake(); + with_share(&mut f, Window { minimised: true, ..win("discord.exe", "게임방", Rect { x: 0, y: 0, w: 960, h: 540 }) }); + assert!(act(&s, &f, &a).unwrap_err().contains("최소화")); + + // Discord on the exclusion list is never seen, popped out or not. + let mut s = settings(); + s.pc_control.screen_exclude.push("Discord".into()); + let mut f = fake(); + with_share(&mut f, win("discord.exe", "게임방", Rect { x: 0, y: 0, w: 960, h: 540 })); + assert_eq!(act(&s, &f, &a).unwrap_err(), NO_SHARE_WINDOW); + } + + #[test] + fn the_popped_out_window_naming_jarvis_voice_channel_goes_first() { + let s = settings(); + let all = vec![ + win("discordptb.exe", "다른 방", Rect { x: 0, y: 0, w: 10, h: 10 }), + win("discordptb.exe", "게임방", Rect { x: 0, y: 0, w: 10, h: 10 }), + win("discordptb.exe", "Discord", Rect { x: 0, y: 0, w: 10, h: 10 }), + ]; + assert_eq!(share_window(&s, &all, Some("게임방")).unwrap().title, "게임방"); + assert_eq!(share_window(&s, &all, None).unwrap().title, "다른 방", "else the one nearest the front"); + assert!(share_window(&s, &all[2..], Some("게임방")).is_none(), "the main window is never the share"); + } + + #[test] + fn a_share_is_seen_only_while_the_person_asked_about_shares_their_screen() { + let streaming = vec![(1, "철수".to_owned()), (2, "Younghee".to_owned())]; + assert_eq!(share_streamer(&streaming, Some("철수님")), Ok(Some("철수".to_owned())), "a spoken suffix still matches"); + assert_eq!(share_streamer(&streaming, Some("younghee")), Ok(Some("Younghee".to_owned()))); + assert_eq!(share_streamer(&streaming, Some("민수")).unwrap_err(), "민수님은 지금 화면공유 중이 아닙니다"); + assert_eq!(share_streamer(&streaming, None), Ok(None), "no name: whatever the window shows"); + assert_eq!(share_streamer(&streaming[..1], Some(" ")), Ok(Some("철수".to_owned()))); + assert!(share_streamer(&[], None).unwrap_err().contains("화면공유 중인 사람이 없습니다")); + assert!(share_streamer(&[], Some("철수")).unwrap_err().contains("철수님은")); + } + + #[test] + fn the_share_prompt_names_the_person_when_it_is_known() { + assert!(share_prompt(Some("철수")).contains("철수님 화면공유")); + assert!(share_prompt(None).contains("화면공유를")); + } + #[test] fn scaling_averages_the_area() { let mut img = solid(4, 2, 0); diff --git a/app/src/worker_client.rs b/app/src/worker_client.rs index cb9d40a..8f7c2a0 100644 --- a/app/src/worker_client.rs +++ b/app/src/worker_client.rs @@ -172,16 +172,25 @@ pub struct ScreenAction { /// `capture` or `list`. pub action: String, pub user: String, - /// capture: `screen`, `monitor` or `window`. + /// `voice` or `chat`. + #[serde(default)] + pub via: Option, + /// capture: `screen`, `monitor`, `window` or `share`. #[serde(default)] pub target: Option, #[serde(default)] pub monitor: Option, #[serde(default)] pub window: Option, + /// share: the name of the person whose screen share is asked about. + #[serde(default)] + pub person: Option, /// `fast`, `normal` or `detailed`. #[serde(default)] pub detail: Option, + /// share: the name of Jarvis's voice channel, set by the core (the popped-out window's title). + #[serde(skip)] + pub voice_channel: Option, } /// The brain's pcControl tool asks the app for the person asking (voice_worker.spec.md, "PC control"). diff --git a/docs/llm_contexts.md b/docs/llm_contexts.md index 0153e24..5b815b7 100644 --- a/docs/llm_contexts.md +++ b/docs/llm_contexts.md @@ -26,7 +26,7 @@ Every distinct LLM call in Jarvis, what feeds it, what consumes it, and how it i - **v2 schedule tool and scheduled tasks**: when the voice worker's brain sets a schedule backend (`turn_state.set_schedule`, every voice and chat reply), `schedule` is in the catalogue for the router (#7) and `prepare_intent()` (not force-offered every turn); without a backend it is not in the catalogue at all ([spec](src/jarvis/tools/builtin/schedule.spec.md)). A scheduled `task` the app fires is one more run of this loop (`reply` `via: "task"`, at the task's time, in the worker's chat lane, waiting for LLM switches like any reply): the asker's history scope, a fresh short-term memory, no prepared pick, no chat or schedule backend, `mcps={}` and `disabled_tools` widened to everything but `TASK_TOOLS` (`getWeather`, `webSearch`, `fetchWebPage`, `fetchMeals`, `collectedNews`, `toolSearchTool`), so the router, planner and loop see only those. The app waits at most 300 s for its answer and delivers it (`app/src/admin.spec.md`, "Schedules"). Eval: `evals/test_schedule_tool.py`. - **v2 personal notes**: the app sends the asker's standing instructions with every voice, chat and task reply; `run_reply_engine(person_notes=...)` adds them to the system message (after the warm profile, before the history block) inside a `` fence, labelled as that person's own answer preferences that never permit an action or a tool. Only the asker's notes, at most 20 of 300 characters. The `personalNotes` tool (add, list, delete) is in the catalogue only with a notes backend (`turn_state.set_notes`, every voice and chat reply; not tasks), also for `prepare_intent()`. No extra LLM call ([spec](src/jarvis/tools/builtin/personal_notes.spec.md)). Eval: `evals/test_personal_notes.py`. - **v2 collected news**: the `collectedNews` tool (list targets, read a target's newest items) puts the background collection's results into the loop inside an UNTRUSTED COLLECTED CONTENT fence; offered only with the app's collection file (`JARVIS_COLLECT_PATH`) and `features.collect` on, and one of the scheduled tasks' tools (`TASK_TOOLS`). No LLM call inside. The app's news notifications post titles and links with no LLM ([spec](src/jarvis/tools/builtin/collected_news.spec.md)). Eval: `evals/test_collected_news.py`. - - **v2 screen view (pictures)**: the `screenView` tool (whole screen, one monitor or one window, or the list of monitors and windows) returns base64 JPEG pictures with its result; the loop puts them on the tool message (`images`) and `llm.py` sends them to Strata as a user message of `image_url` parts right after the tool messages (about 768 / 1280 px long side, or four parts for `detailed`; at most 3 MiB). Offered only when the app makes screen viewing available (Windows, someone registered for PC control), which also starts the LLM with `--vision gpu` (normal stage) or `cpu` (game stage), else `--vision off`. Pictures stay in the one reply: not in the tool carryover, logs or prompt dumps. No extra LLM call ([spec](src/jarvis/tools/builtin/screen_view.spec.md)). Eval: `evals/test_screen_view_tool.py`. + - **v2 screen view (pictures)**: the `screenView` tool (whole screen, one monitor, one window or someone's Discord screen share in its popped-out window, or the list of monitors and windows) returns base64 JPEG pictures with its result; the loop puts them on the tool message (`images`) and `llm.py` sends them to Strata as a user message of `image_url` parts right after the tool messages (about 768 / 1280 px long side, or four parts for `detailed`; at most 3 MiB). Offered only when the app makes screen viewing available (Windows, someone registered for PC control), which also starts the LLM with `--vision gpu` (normal stage) or `cpu` (game stage), else `--vision off`. Pictures stay in the one reply: not in the tool carryover, logs or prompt dumps. No extra LLM call ([spec](src/jarvis/tools/builtin/screen_view.spec.md)). Eval: `evals/test_screen_view_tool.py`. - Tool schema: native via `generate_tools_json_schema()` ([src/jarvis/tools/registry.py](src/jarvis/tools/registry.py)) or text fallback via `_text_tool_call_guidance()` ([engine.py:68](src/jarvis/reply/engine.py:68)) - Tool results from prior turns (raw or digested — see #5) - **Output**: OpenAI-style `{content, tool_calls, thinking}`. Consumed by the tool orchestrator and TTS pipeline. Natural-language content is delivered immediately; no post-turn evaluator runs. Spoken-answer length: the persona (`system_prompt.py`) and `voice_style` (`prompts/system.py`) both constrain the reply to a SINGLE sentence — any dry aside must fold into that one sentence as a trailing clause, never a second sentence. This keeps TTS latency down (synth time scales with text length) and matches the `agents/llm.md` operator instruction. diff --git a/docs/v2/pc-features.md b/docs/v2/pc-features.md index 22d576d..f1e3ff5 100644 --- a/docs/v2/pc-features.md +++ b/docs/v2/pc-features.md @@ -319,6 +319,8 @@ LLM과 STT를 함께 켠 측정(M0 `gamemode_stt`, 게임 없음). STT를 먼저 최소화하면 캡처가 안 되므로 그때는 창을 다시 열어 달라고 안내합니다. +**지금 구현(M7 3단계 둘째 부분, 2026-11-19)**: `screenView`의 `share` 대상(사람 이름은 선택), 앱 `app/src/screen.rs`와 `discord.rs`([admin.spec.md](../../app/src/admin.spec.md) "Screen viewing"). 분리 창은 제목이 "Discord"로 끝나지 않는 디스코드 창으로 찾고, 자비스가 있는 음성 채널 이름이 제목에 든 창을 먼저 씁니다. 창이 없으면 음성 채널에 "디스코드에서 철수님 화면공유를 시청하고 창 분리 버튼을 눌러 주세요."라고 말하고 1초마다 30초 동안 찾습니다(채팅으로 물으면 기다리지 않고 같은 안내로 답함). 대기 시간은 아직 설정 화면에 두지 않았습니다. 실제 디스코드 분리 창 제목과 실제 모델(Qwen3.6)의 도구 고르기는 게임PC에서 확인할 것. 작은 모델(gemma3:4b, qwen3:8b)로 돌린 평가에서는 "철수 화면 뭐 하고 있어?"에 `share`를 고르는 일이 드물었습니다. + ## 백그라운드 크로미움으로 정보 수집 - 봇용 크롬(데이터 폴더 안)을 크롬 개발자 프로토콜(CDP)로 조종합니다. 웹 검색, 페이지 읽기, 예약 수집에 씁니다. diff --git a/evals/test_screen_view_tool.py b/evals/test_screen_view_tool.py index d9c3d01..5eec8fc 100644 --- a/evals/test_screen_view_tool.py +++ b/evals/test_screen_view_tool.py @@ -6,6 +6,8 @@ Eval: seeing the screen (src/jarvis/tools/builtin/screen_view.spec.md). - When the app refuses (the person is not registered), the answer says it cannot see the screen and does not describe one. - A question that is not about the screen does not capture it. +- "철수 화면 뭐 하고 있어?" asks for 철수's Discord screen share (target share, person 철수), and + the answer reads that picture. The screen backend is a fake that returns a fixed picture; the tool, the reply loop and the picture transport (llm.py) run for real. Needs a model that reads pictures: @@ -38,8 +40,8 @@ class FakeScreen: {"number": 2, "width": 1280, "height": 720, "main": False}], "windows": [{"program": "notepad", "title": "Untitled - Notepad", "minimised": False}]}} - return {"ok": True, "item": {"what": f"모니터 {kw.get('monitor', 2)}", "width": 1280, "height": 720, - "parts": 1}, "images": [PICTURE]} + what = "화면공유" if kw.get("target") == "share" else f"모니터 {kw.get('monitor', 2)}" + return {"ok": True, "item": {"what": what, "width": 1280, "height": 720, "parts": 1}, "images": [PICTURE]} def _run(mock_config, eval_db, eval_dialogue_memory, query, screen): @@ -84,3 +86,11 @@ class TestScreenView: screen = FakeScreen() _run(mock_config, eval_db, eval_dialogue_memory, "고양이는 하루에 몇 시간 자?", screen) assert not any(a == "capture" for a, _ in screen.acts), screen.acts + + def test_someones_screen_share_is_asked_for_by_their_name(self, mock_config, eval_db, eval_dialogue_memory): + screen = FakeScreen() + reply = _run(mock_config, eval_db, eval_dialogue_memory, "철수 화면 뭐 하고 있어?", screen) + captures = [kw for a, kw in screen.acts if a == "capture"] + assert captures and captures[0].get("target") == "share" and "철수" in (captures[0].get("person") or ""), \ + screen.acts + assert "BANANA" in reply.upper() or "바나나" in reply or "42" in reply, reply diff --git a/src/jarvis/tools/builtin/screen_view.py b/src/jarvis/tools/builtin/screen_view.py index c93a2fa..ad709b4 100644 --- a/src/jarvis/tools/builtin/screen_view.py +++ b/src/jarvis/tools/builtin/screen_view.py @@ -10,9 +10,10 @@ from ..base import Tool, ToolContext from ..types import ToolExecutionResult from ...debug import debug_log -_TARGETS = ("screen", "monitor", "window", "list") +_TARGETS = ("screen", "monitor", "window", "share", "list") _DETAILS = ("fast", "normal", "detailed") MAX_WINDOW_CHARS = 200 +MAX_PERSON_CHARS = 200 def _fail(text: str) -> ToolExecutionResult: @@ -37,10 +38,11 @@ class ScreenViewTool(Tool): def description(self) -> str: return ( "See this PC's screen as a picture: target='screen' (all monitors), 'monitor' with its " - "number, or 'window' with part of the program's name or window title; target='list' " - "names the monitors and open windows. Use it whenever the person asks what is on their " - "screen, a monitor or a program's window; detail='detailed' to read small text. Answer " - "only from the picture." + "number, 'window' with part of the program's name or window title, or 'share' for " + "another person's screen, which they share on Discord (person: their name); " + "target='list' names the monitors and open windows. Use it whenever the person asks " + "what is on their screen, a monitor, a program's window or someone else's screen; " + "detail='detailed' to read small text. Answer only from the picture." ) @property @@ -51,6 +53,7 @@ class ScreenViewTool(Tool): "target": {"type": "string", "enum": list(_TARGETS)}, "monitor": {"type": "integer", "description": "monitor: its number (1, 2, ...)"}, "window": {"type": "string", "description": "window: part of its program name or title"}, + "person": {"type": "string", "description": "share: the name of the person sharing"}, "detail": {"type": "string", "enum": list(_DETAILS)}, }, "required": [], @@ -80,6 +83,12 @@ class ScreenViewTool(Tool): if not w or len(w) > MAX_WINDOW_CHARS: return _fail("window needs part of the program name or title; call target='list' to see them") fields["window"] = w + if target == "share": + p = " ".join(str(args.get("person") or "").split()) + if len(p) > MAX_PERSON_CHARS: + return _fail("person needs the name of the person sharing their screen") + if p: + fields["person"] = p fields["detail"] = _word(args.get("detail"), _DETAILS, "normal") result = backend.act("capture", **fields) or {} images = [i for i in result.get("images") or [] if isinstance(i, str) and i] diff --git a/src/jarvis/tools/builtin/screen_view.spec.md b/src/jarvis/tools/builtin/screen_view.spec.md index 5728bf6..7a4a7d9 100644 --- a/src/jarvis/tools/builtin/screen_view.spec.md +++ b/src/jarvis/tools/builtin/screen_view.spec.md @@ -32,6 +32,7 @@ Lets Jarvis see this PC's screen: all monitors as one picture, one monitor, or o | `screen` (default) | | every monitor as one picture | | `monitor` | `monitor` (its number, as Windows numbers it) | that monitor | | `window` | `window` (part of its title or program name) | that window, also when other windows cover it | +| `share` | `person` (optional: the name of the person sharing, as said) | the Discord screen share someone streams in Jarvis's voice channel, from the window Discord pops it out to | | `list` | | the monitors (number, size, main) and the open windows (program, title), no picture | `detail`: `fast` (long side about 768 px), `normal` (default, about 1280 px) or `detailed` (the @@ -44,6 +45,9 @@ as their first known word; unknown values fall back to the defaults. - `act("capture", target=..., monitor=..., window=..., detail=...)` gives `{"ok": true, "item": {"what", "width", "height", "parts"}, "images": ["", ...]}`. +- `act("capture", target="share", person=..., detail=...)` gives the same; `person` is sent only + when given (at most 200 characters). The app checks that the person shares their screen and may + wait up to 30 s while it asks for the share to be popped out (admin.spec.md, "Screen viewing"). - `act("list")` gives `{"ok": true, "item": {"monitors": [...], "windows": [...]}}`. - Failures are `{"ok": false, "error": "..."}`. diff --git a/src/voice_worker/server.py b/src/voice_worker/server.py index 2e4a946..bf8c1c4 100644 --- a/src/voice_worker/server.py +++ b/src/voice_worker/server.py @@ -169,8 +169,9 @@ class VoiceWorker: def action_timeout_s(self, kind: str) -> float: """How long an app action is waited for: a PC action (starting a program and reading its new - window, admin.spec.md "PC control") gets at least PC_ACTION_TIMEOUT_S.""" - return max(self.chat_timeout_s, PC_ACTION_TIMEOUT_S) if kind == "pc" else self.chat_timeout_s + window, admin.spec.md "PC control") and a screen action (a screen share waited for while it + is popped out, "Screen viewing") get at least PC_ACTION_TIMEOUT_S.""" + return max(self.chat_timeout_s, PC_ACTION_TIMEOUT_S) if kind in ("pc", "screen") else self.chat_timeout_s async def start(self) -> None: if self.history is not None: diff --git a/src/voice_worker/voice_worker.spec.md b/src/voice_worker/voice_worker.spec.md index 5879b39..b7065c7 100644 --- a/src/voice_worker/voice_worker.spec.md +++ b/src/voice_worker/voice_worker.spec.md @@ -436,7 +436,8 @@ captures and decides who may see (`app/src/admin.spec.md`, "Screen viewing"): ``` <- {"type": "screen_action", "id": "s1", "action": "capture|list", "user": "", "via": "voice|chat", - "channel": "...", "target": "screen|monitor|window", "monitor": 2, "window": "chrome", "detail": "fast|normal|detailed"} + "channel": "...", "target": "screen|monitor|window|share", "monitor": 2, "window": "chrome", "person": "철수", + "detail": "fast|normal|detailed"} -> {"type": "screen_action_result", "id": "s1", "ok": true, "item": {...}, "images": [""]} // or ok false with "error" ``` @@ -444,6 +445,8 @@ captures and decides who may see (`app/src/admin.spec.md`, "Screen viewing"): waiting are those of chat and schedule actions. `images` that are not a list of strings are dropped. The pictures stay in that one reply (screen_view.spec.md). - Messages may be up to 4 MiB (pictures), the WebSocket's limit. +- A screen action is waited for as long as a PC action (45 s): the app may wait up to 30 s for a + screen share to be popped out. ### PC control (M7) diff --git a/tests/test_screen_view.py b/tests/test_screen_view.py index 3f2b225..9a7833a 100644 --- a/tests/test_screen_view.py +++ b/tests/test_screen_view.py @@ -68,6 +68,15 @@ def test_a_window_is_found_by_part_of_its_name(app): assert app.calls[-1] == ("capture", {"target": "window", "window": "크롬", "detail": "normal"}) +@pytest.mark.unit +def test_a_discord_screen_share_is_asked_for_with_the_person_named(app): + assert run(target="share", person=" 철수 ").success + assert run(target="share").success + assert app.calls == [("capture", {"target": "share", "person": "철수", "detail": "normal"}), + ("capture", {"target": "share", "detail": "normal"})] + assert not run(target="share", person="x" * 201).success and len(app.calls) == 2 + + @pytest.mark.unit @pytest.mark.parametrize("args", [{"target": "monitor"}, {"target": "monitor", "monitor": 0}, {"target": "monitor", "monitor": "two"}, {"target": "window"}, diff --git a/tests/test_voice_worker_brain.py b/tests/test_voice_worker_brain.py index f8b6ff8..ad89647 100644 --- a/tests/test_voice_worker_brain.py +++ b/tests/test_voice_worker_brain.py @@ -1652,26 +1652,32 @@ def test_a_pc_action_goes_to_the_app_and_an_ask_answer_keeps_what_to_ask(store): @pytest.mark.unit -def test_a_pc_action_waits_longer_than_the_other_app_actions(store): +def test_pc_and_screen_actions_wait_longer_than_the_other_app_actions(store): """Starting a program and reading its new window takes the app up to about 20 s (admin.spec.md, - "PC control"): a PC action is not given up after the chat actions' wait.""" + "PC control"), and a screen share is waited for up to 30 s while it is popped out ("Screen + viewing"): those actions are not given up after the chat actions' wait.""" results = [] - brain = FakeBrain(lambda u, t: results.append((brain.screen.act("list"), brain.pc.act("run", program="calc"))) - or "ok") + brain = FakeBrain(lambda u, t: results.append((brain.notes_backend.act("list"), + brain.screen.act("capture", target="share"), + brain.pc.act("run", program="calc"))) or "ok") async def body(url): async with connect(url) as ws: await hello(ws) await send(ws, type="reply", id="r1", user="100", text="q", via="voice", speak=False) - await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "screen_action") - seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "pc_action") + await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "note_action") + seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "screen_action") await asyncio.sleep(0.6) # longer than chat_timeout_s + await send(ws, type="screen_action_result", id=seen[-1]["id"], ok=True, item={"what": "화면공유"}, + images=["x"]) + seen = await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "pc_action") + await asyncio.sleep(0.6) await send(ws, type="pc_action_result", id=seen[-1]["id"], ok=True, item={"window": {"id": "w1"}}) await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text") asyncio.run(with_worker(body, store=store, brain=brain, chat_timeout_s=0.3)) - (screen, pc), = results - assert not screen["ok"] and pc == {"ok": True, "item": {"window": {"id": "w1"}}} + (note, screen, pc), = results + assert not note["ok"] and screen["ok"] and pc == {"ok": True, "item": {"window": {"id": "w1"}}} @pytest.mark.unit