diff --git a/app/src/admin.spec.md b/app/src/admin.spec.md index 4389080..2db8b10 100644 --- a/app/src/admin.spec.md +++ b/app/src/admin.spec.md @@ -662,10 +662,14 @@ start it first, so nothing answers before the brain can, and stop it last. - **A server the app does not start**: with `llm.url` (an OpenAI-compatible `/v1` address, in the settings file only) the app starts nothing; that server must list a model within 30 seconds. - **Settings**: `llm.context` (default 32768 tokens per request, 4096-262144) and `llm.port` - (default 8080) on the model screen; `llm.url` (default none). + (default 8080) on the model screen; `llm.url` (default none). `brain.location` (기본 위치, on the + model screen, default `대한민국`, up to 100 characters, empty: the brain detects it): where the + people Jarvis answers are, for the time line, weather without a place and searches + (`src/jarvis/utils/location.spec.md`). - **The brain's settings**: on every start the app writes `config/brain.json` (it owns the file): `ollama_base_url` the LLM's `/v1`, `ollama_chat_model` the listed id, `db_path` - `memory/jarvis.db`, `disabled_tools` and `mcps` (above, "Features and MCP servers"); the worker gets it as `JARVIS_CONFIG_PATH`. Loading it writes nothing to the + `memory/jarvis.db`, `disabled_tools` and `mcps` (above, "Features and MCP servers"), + `location_name` (`brain.location`, null when empty); the worker gets it as `JARVIS_CONFIG_PATH`. Loading it writes nothing to the home folder. ## Chat channel names and topics (`app/src/channel_sync.rs`) diff --git a/app/src/settings.rs b/app/src/settings.rs index ca90019..1b60aac 100644 --- a/app/src/settings.rs +++ b/app/src/settings.rs @@ -226,8 +226,14 @@ pub struct Settings { /// The window's close button (X) hides it to the tray instead of quitting (`app.closeButton`: /// "tray" or "quit", default "quit"; admin.spec.md, "Window and Windows build"). pub close_to_tray: bool, + /// Where the people Jarvis answers are, by default (`brain.location`, default "대한민국"): the + /// brain's location for the time line, weather without a place and searches (admin.spec.md, + /// "LLM"); None (an empty value) leaves it to the brain's own detection. + pub location: Option, } +pub const DEFAULT_LOCATION: &str = "대한민국"; + #[derive(Debug, Clone, PartialEq, Eq)] pub struct SettingsError(pub String); @@ -279,6 +285,7 @@ impl Default for Settings { type_text: Level::Ask, run: Level::Ask, screen_exclude: ["1Password", "Bitwarden", "KeePass", "KeePassXC"].map(String::from).into() }, close_to_tray: false, + location: Some(DEFAULT_LOCATION.into()), } } } @@ -632,6 +639,13 @@ pub fn parse_settings(raw: &Value) -> Result { "quit" => false, _ => return err("app.closeButton must be tray or quit"), }, + location: { + let l = get_str(section(raw, "brain")?, "location", "brain", DEFAULT_LOCATION)?.trim().to_owned(); + if l.chars().count() > 100 { + return err("brain.location must be at most 100 characters"); + } + (!l.is_empty()).then_some(l) + }, }) } @@ -1006,6 +1020,12 @@ mod tests { #[test] fn the_close_button_choice_is_read_and_checked() { assert!(!parse_settings(&json!({})).unwrap().close_to_tray, "closing quits by default"); + assert_eq!(parse_settings(&json!({})).unwrap().location.as_deref(), Some(DEFAULT_LOCATION)); + assert_eq!(parse_settings(&json!({"brain": {"location": " 부산 "}})).unwrap().location.as_deref(), Some("부산")); + assert_eq!(parse_settings(&json!({"brain": {"location": " "}})).unwrap().location, None, "empty: the brain detects it"); + for bad in [json!({"brain": {"location": 1}}), json!({"brain": {"location": "가".repeat(101)}}), json!({"brain": 1})] { + assert!(parse_settings(&bad).unwrap_err().0.contains("brain"), "{bad}"); + } assert!(parse_settings(&json!({"app": {"closeButton": "tray"}})).unwrap().close_to_tray); assert!(!parse_settings(&json!({"app": {"closeButton": "quit"}})).unwrap().close_to_tray); for bad in [json!({"app": {"closeButton": "hide"}}), json!({"app": {"closeButton": 1}}), json!({"app": 1})] { diff --git a/app/src/settings_edit.rs b/app/src/settings_edit.rs index 09ae9db..0b1fdab 100644 --- a/app/src/settings_edit.rs +++ b/app/src/settings_edit.rs @@ -128,6 +128,8 @@ pub fn fields() -> Vec { f("voice.tts.speed", Model, Number { min: 0.5, max: 2.0 }, json!(1.5), "말하기 속도(1.0 보통, 기본 1.5, 저장하면 바로 적용)"), // The LLM (admin.spec.md, "LLM"); `llm.url` (a server the app does not start) is file-only. f("llm.context", Model, Int { min: 4096, max: 262144 }, json!(32768), "대화 모델이 한 번에 기억하는 양(토큰)"), + f("brain.location", Model, Text { max: 100, non_empty: false }, json!(crate::settings::DEFAULT_LOCATION), + "기본 위치(날씨, 검색, 추천의 기준, 비우면 자동 감지)"), f("llm.port", Model, Int { min: 1024, max: 65535 }, json!(8080), "대화 모델 포트"), // The brain's features (admin.spec.md, "Features and MCP servers"). f("features.weather", Features, Bool, json!(true), "날씨"), diff --git a/app/src/strata.rs b/app/src/strata.rs index 5630f3f..a32a2c9 100644 --- a/app/src/strata.rs +++ b/app/src/strata.rs @@ -250,6 +250,7 @@ pub fn write_brain_config(data: &Path, base_url: &str, model: &str, settings: &c "db_path": data.join("memory").join("jarvis.db").to_string_lossy(), "disabled_tools": crate::features::disabled_tools(settings), "mcps": crate::features::brain_mcps(&settings.mcp_servers), + "location_name": settings.location, "_config_version": 1, }); let mut text = serde_json::to_string_pretty(&cfg).unwrap(); @@ -716,6 +717,11 @@ pub(crate) mod tests { assert!(Path::new(v["db_path"].as_str().unwrap()).starts_with(&l.data)); assert_eq!(v["disabled_tools"], json!(crate::features::disabled_tools(&s))); assert_eq!(v["mcps"], crate::features::brain_mcps(&s.mcp_servers)); + assert_eq!(v["location_name"], crate::settings::DEFAULT_LOCATION, "the people are in Korea by default"); + let none = parse_settings(&json!({"brain": {"location": ""}})).unwrap(); + let f = write_brain_config(&l.data, "http://127.0.0.1:8080/v1", "m", &none).unwrap(); + let v: Value = serde_json::from_str(&std::fs::read_to_string(f).unwrap()).unwrap(); + assert_eq!(v["location_name"], Value::Null); } #[cfg(unix)] diff --git a/docs/llm_contexts.md b/docs/llm_contexts.md index 772b50b..886b01e 100644 --- a/docs/llm_contexts.md +++ b/docs/llm_contexts.md @@ -164,7 +164,7 @@ Every distinct LLM call in Jarvis, what feeds it, what consumes it, and how it i - **Trigger**: once per reply, **after the tool router and before memory search**. **v2 brain** (`intent_judge=True`): runs only on turns where the router picked a tool (not for the sentinels `stop` or `toolSearchTool` alone; a router-picked `chatChannels` counts), so a turn without tools has one LLM call before its answer. Skipped when `cfg.planner_enabled = False`, when the query is shorter than `MIN_QUERY_CHARS` (4), or when no model / base URL is available. - **Model / gating**: resolution chain `planner_model (override) → ollama_chat_model`. The planner tracks the chat model so upgrading the chat model (via setup wizard or config) automatically upgrades plan quality. - **Inputs**: user query, dialogue context, the live time/location line (`LIVE CONTEXT:`, so a location-dependent ask plans `searchMemory topic='where the user lives'` only when the location is unknown or disabled), **router-narrowed** tool catalogue (names + one-line descriptions + argument names, built by `planner_tool_catalog()`) — not the full 30+ list. When the carry-over guard from #7 fires, the previous turn's failed tool name is unioned into this catalogue before the planner sees it, so the planner can plan a re-call without `toolSearchTool` round-tripping. **No** memory context — the planner decides *whether* memory is needed. -- **System prompt**: `_PROMPT_TEMPLATE` in `planner.py`. Teaches the `searchMemory topic='...'` directive for prior-conversation lookups, short imperative tool steps, angle-bracket entity placeholders, final synthesis step, same-language output, no numbering. +- **System prompt**: `_PROMPT_TEMPLATE` in `planner.py`. Teaches the `searchMemory topic='...'` directive for prior-conversation lookups, short imperative tool steps, angle-bracket entity placeholders, final synthesis step, same-language output (the words of search and play arguments too, repeated in the user message's closing line), no numbering. - **Output**: list of plan steps (max `MAX_STEPS` = 5). Gates memory enrichment (#3 / #4) and augments the tool router (#7 — planner's picks are unioned in, not replacing). Single-step `["Reply to the user."]` plans are the planner's positive "no memory, no tools" signal. An empty list is fail-open — the engine reverts to running #3 unconditionally. Consumed further by the engine to build the `ACTION PLAN:` system-message block and drive the direct-exec loop (#13) for small models. - **Limits**: `planner_timeout_sec` (6s). Fail-open → `[]`. diff --git a/evals/test_bot_browser_tool.py b/evals/test_bot_browser_tool.py index 70ca834..3e4beed 100644 --- a/evals/test_bot_browser_tool.py +++ b/evals/test_bot_browser_tool.py @@ -18,6 +18,8 @@ Cases: Run: EVAL_JUDGE_MODEL=qwen2.5:7b python -m pytest evals/test_bot_browser_tool.py -v """ +import re + import pytest from conftest import requires_judge_llm @@ -88,6 +90,15 @@ class YoutubeSession(FakeSession): self.url = "https://www.youtube.com/results?search_query=x" return dict(self.RESULTS) + def play(self, results_url): + # The real session opens the first video and starts its player (tests/test_bot_browser.py). + import urllib.parse + self.calls.append(("play", results_url)) + self.url = "https://www.youtube.com/watch?v=nP1xjFX86Po" + self.playing = True + words = urllib.parse.unquote_plus(results_url.split("search_query=")[-1]) + return dict(self._page(self.url), title=f"{words} (Official MV) - YouTube", text=f"{words} (Official MV)") + def click(self, target): self.calls.append(("click", target)) if "watch" in getattr(self, "url", ""): @@ -148,17 +159,34 @@ class TestBotBrowserTool: import urllib.parse s, _ = _run(monkeypatch, mock_config, eval_db, eval_dialogue_memory, "유튜브에서 아이유 신곡 틀어 줘", session=YoutubeSession()) - words = [urllib.parse.unquote_plus(c[1]) for c in s.calls if c[0] == "open"] + \ + words = [urllib.parse.unquote_plus(c[1]) for c in s.calls if c[0] in ("open", "play")] + \ [c[1] for c in s.calls if c[0] == "type"] - assert any("youtube" in urllib.parse.unquote_plus(c[1]).lower() for c in s.calls if c[0] == "open"), s.calls + assert any("youtube" in urllib.parse.unquote_plus(c[1]).lower() for c in s.calls if c[0] in ("open", "play")), s.calls assert any("아이유" in w or "iu" in w.lower() for w in words), s.calls assert not any("아이우" in w for w in words), s.calls + @pytest.mark.parametrize("query,said", [("유튜브에서 아이유 신곡 틀어 줘", ("아이유", "iu")), + ("자비스, 유튜브에서 뉴진스 하입보이 틀어 줘", ("뉴진스", "newjeans"))]) + def test_playing_a_song_ends_on_a_playing_video_found_with_the_words_as_said( + self, monkeypatch, mock_config, eval_db, eval_dialogue_memory, query, said): + # Live 2026-11-16 (0.1.19): the model stopped on the results page or on YouTube's home page + # (a third browser call was refused), searched "NewJeans Hypothesis" in English, and still + # said the video played. + import urllib.parse + s, reply = _run(monkeypatch, mock_config, eval_db, eval_dialogue_memory, query, session=YoutubeSession()) + assert s.playing and "watch" in s.url, f"{s.calls} reply={reply!r}" + searched = [urllib.parse.unquote_plus(c[1]) for c in s.calls if c[0] in ("open", "play")] + \ + [c[1] for c in s.calls if c[0] == "type"] + # The words in the language they were said in (a Korean request searches in Korean, so the + # results are the ones people there get), naming who was asked for. + assert any(re.search(r"[가-힣]", w) and any(n in w.lower() for n in said) for w in searched), s.calls + def test_a_video_page_that_stays_paused_is_played_or_reported_as_not_playing( self, monkeypatch, mock_config, eval_db, eval_dialogue_memory): s, reply = _run(monkeypatch, mock_config, eval_db, eval_dialogue_memory, "브라우저로 https://www.youtube.com/watch?v=nP1xjFX86Po 열어서 영상 재생해 줘", session=YoutubeSession()) - assert any(c[0] == "open" and "watch" in c[1] for c in s.calls), s.calls + # It reached the video (opened or played by its address or its ID). + assert "watch" in getattr(s, "url", ""), s.calls # Either it pressed play, or it told the user honestly that the video did not play (a # negation in the Korean reply); never "it plays" over a paused video. assert s.playing or any(n in (reply or "") for n in ("않", "못")), f"{s.calls} reply={reply!r}" diff --git a/evals/test_search_in_users_language.py b/evals/test_search_in_users_language.py new file mode 100644 index 0000000..a9edca4 --- /dev/null +++ b/evals/test_search_in_users_language.py @@ -0,0 +1,54 @@ +""" +Eval: a Korean request searches the web in Korean, so the results are the ones people in Korea get. + +Live 2026-11-16 (0.1.19): "요즘 볼 만한 넷플릭스 드라마 세 개만 추천해 줘" searched "best Netflix dramas to +watch now 2026" and recommended American shows; "엔비디아 rtx 5050 그래픽카드 가격 검색해 줘" searched in +English and answered with an Amazon US price. The reply engine runs as the v2 brain runs it (compact +prompt, Korean default, the place set in the settings: brain.location, default 대한민국); webSearch is +a fake that records its query. Checked by script: the query has Hangul. + +Run: EVAL_JUDGE_MODEL=qwen3:8b python -m pytest evals/test_search_in_users_language.py -v +""" + +import re +from unittest.mock import patch + +import pytest + +from conftest import requires_judge_llm +from helpers import ToolCallCapture, create_mock_tool_run + +CASES = [ + ("netflix", "요즘 볼 만한 넷플릭스 드라마 세 개만 추천해 줘"), + ("price", "엔비디아 RTX 5050 그래픽카드 가격 검색해 줘"), + ("restaurants", "이번 주말에 갈 만한 맛집 좀 찾아 줘"), +] + + +@pytest.mark.eval +@requires_judge_llm +class TestSearchInUsersLanguage: + @pytest.mark.parametrize("case", CASES, ids=[c[0] for c in CASES]) + def test_a_korean_request_searches_in_korean(self, mock_config, eval_db, eval_dialogue_memory, + monkeypatch, case): + from helpers import JUDGE_BASE_URL, JUDGE_MODEL + from jarvis.memory.history import HistoryContext + from jarvis.reply.engine import run_reply_engine + + cid, query = case + monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False) + mock_config.ollama_base_url = JUDGE_BASE_URL + mock_config.ollama_chat_model = JUDGE_MODEL + mock_config.location_name = "대한민국" + capture = ToolCallCapture() + result = "Search results: (fake) 1. 결과 하나 2. 결과 둘" + with patch("jarvis.reply.engine.run_tool_with_retries", + side_effect=create_mock_tool_run(capture, {"webSearch": result})): + reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query, + dialogue_memory=eval_dialogue_memory, history=HistoryContext(), + legacy_memory=False, default_language="Korean", compact_prompt=True) + searches = [str(c["args"].get("search_query") or c["args"].get("query") or "") + for c in capture.calls if c["name"] == "webSearch"] + print(f"\n search language [{JUDGE_MODEL}] {cid}: searches={searches} reply={(reply or '')[:120]!r}") + assert searches, f"[{cid}] no web search: {capture.tool_names()}" + assert all(re.search(r"[가-힣]", q) for q in searches), f"[{cid}] searched in another language: {searches}" diff --git a/src/jarvis/config.py b/src/jarvis/config.py index 43457f9..4a836db 100644 --- a/src/jarvis/config.py +++ b/src/jarvis/config.py @@ -238,6 +238,8 @@ class Settings: location_ip_address: str | None location_auto_detect: bool location_cgnat_resolve_public_ip: bool + # A place the user set (e.g. "대한민국"): used instead of detection when set (location.spec.md). + location_name: str | None # Web Search web_search_enabled: bool @@ -551,6 +553,7 @@ def get_default_config() -> Dict[str, Any]: # When behind CGNAT (100.64.0.0/10), attempt a privacy-light external DNS query to discover true public IP. # Uses a single OpenDNS resolver lookup of myip.opendns.com over DNS (no HTTP services). Disable to avoid any external request. "location_cgnat_resolve_public_ip": True, + "location_name": None, # Web Search "web_search_enabled": True, @@ -762,6 +765,7 @@ def load_settings() -> Settings: location_ip_address = None if location_ip_address_val in (None, "", "null") else str(location_ip_address_val) location_auto_detect = bool(merged.get("location_auto_detect", True)) location_cgnat_resolve_public_ip = bool(merged.get("location_cgnat_resolve_public_ip", True)) + location_name = str(merged.get("location_name") or "").strip() or None web_search_enabled = bool(merged.get("web_search_enabled", True)) brave_search_api_key = str(merged.get("brave_search_api_key", "") or "").strip() wikipedia_fallback_enabled = bool(merged.get("wikipedia_fallback_enabled", True)) @@ -902,6 +906,7 @@ def load_settings() -> Settings: location_ip_address=location_ip_address, location_auto_detect=location_auto_detect, location_cgnat_resolve_public_ip=location_cgnat_resolve_public_ip, + location_name=location_name, # Web Search web_search_enabled=web_search_enabled, diff --git a/src/jarvis/reply/engine.py b/src/jarvis/reply/engine.py index 217d7c9..ec97516 100644 --- a/src/jarvis/reply/engine.py +++ b/src/jarvis/reply/engine.py @@ -761,6 +761,9 @@ def _live_time_location_string(cfg) -> str: tz_name: Optional[str] = None if not getattr(cfg, 'location_enabled', True): location_context = "Location: Disabled" + elif isinstance(getattr(cfg, 'location_name', None), str) and cfg.location_name.strip(): + # The place the user set wins over detection (location.spec.md); local time is the PC's. + location_context = f"Location: {cfg.location_name.strip()} (where the user is)" else: location_context, tz_name = get_location_context_with_timezone( config_ip=getattr(cfg, 'location_ip_address', None), diff --git a/src/jarvis/reply/planner.py b/src/jarvis/reply/planner.py index 012d455..b71568c 100644 --- a/src/jarvis/reply/planner.py +++ b/src/jarvis/reply/planner.py @@ -232,7 +232,10 @@ _PROMPT_TEMPLATE = ( "10. Maximum {max_steps} steps. Do not number them — one step per line.\n" "11. Output ONLY the steps, no preamble, no trailing commentary, no " "JSON fences, no explanations.\n" - "12. Write the steps in the same language the user wrote the query in.\n" + "12. Write the steps in the same language the user wrote the query in. " + "The words of every search or play argument stay in the user's " + "language too, never translated into English, because a search in the " + "user's language finds what people where they are get.\n" ) @@ -279,7 +282,8 @@ def _build_user_message( else: parts.append("DIALOGUE CONTEXT: (empty)") parts.append(f"USER QUERY: {query.strip()}") - parts.append("\nEmit the plan now, one step per line, no numbering.") + parts.append("\nEmit the plan now, one step per line, no numbering, written in the language of " + "the USER QUERY (the words of searches too).") return "\n\n".join(parts) diff --git a/src/jarvis/reply/planner.spec.md b/src/jarvis/reply/planner.spec.md index 85203eb..b0b5f37 100644 --- a/src/jarvis/reply/planner.spec.md +++ b/src/jarvis/reply/planner.spec.md @@ -108,7 +108,14 @@ The planner prompt instructs the model to emit: `webSearch query='his most famous songs'`. - A final synthesis/reply step when any `searchMemory` or tool step was planned. -- Steps in the same language the user wrote the query in. +- Steps in the same language the user wrote the query in, the words of + search and play arguments too (never translated into English: a search + in the user's language finds what people where they are get); the + closing line of the user message repeats it, which a small model follows + better than the rule alone. Eval `evals/test_search_in_users_language.py` + (qwen3:8b, 3 runs of 3 Korean requests): Korean searches 6/9 before + ("요즘 볼 만한 넷플릭스 드라마" searched "best netflix dramas 2026" 3/3), 8/9 + after (that one 2/3). ### Parsing and hygiene diff --git a/src/jarvis/tools/builtin/bot_browser.py b/src/jarvis/tools/builtin/bot_browser.py index 847f960..368700e 100644 --- a/src/jarvis/tools/builtin/bot_browser.py +++ b/src/jarvis/tools/builtin/bot_browser.py @@ -18,8 +18,12 @@ from ...debug import debug_log FENCE_START = "<<>>" FENCE_END = "<<>>" -_ACTIONS = ("open", "read", "click", "type", "back") +_ACTIONS = ("open", "read", "click", "type", "back", "play") PAGE_WAIT_S = 15.0 +# play: two pages (the results and the video) and the wait for the video to start. +PLAY_WAIT_S = 25.0 +# Where play searches: YouTube's results for the words (encoded by the tool). +PLAY_SEARCH = "https://www.youtube.com/results?search_query=" TEXT_CHARS = 6000 MAX_ITEMS = 40 CDP_ENV = "JARVIS_BROWSER_CDP" @@ -58,6 +62,12 @@ _PLAY = r""" (() => { const v = [...document.querySelectorAll('video')].filter(m => (m.currentSrc || m.src) && m.paused && !m.ended && !m.error && m.currentTime === 0 && m.getBoundingClientRect().width >= innerWidth * 0.5); if (!v.length) return false; v[0].play().catch(() => {}); return true; })()""" +# The first video in a list of results (a visible `/watch?v=` link, not an ad), as its plain address. +_FIRST_VIDEO = "(() => {" + _HELPERS + r""" +const a = [...document.querySelectorAll('a[href*="/watch?v="]')].find(e => vis(e) + && !e.closest('ytd-ad-slot-renderer,ytd-promoted-video-renderer,ytd-in-feed-ad-layout-renderer,[is-ad]')); +if (!a) return null; const u = new URL(a.getAttribute('href'), location.href); +return u.origin + u.pathname + '?v=' + encodeURIComponent(u.searchParams.get('v')); })()""" _CLICK = "(() => {" + _HELPERS + r""" return ((t) => { const m = /^\s*\[(\d+)\]\s*$/.exec(t); let el = null; if (m) el = (window.__jvItems || [])[Number(m[1]) - 1]; @@ -239,6 +249,18 @@ class BrowserSession: self._press_play() return self.read() + def play(self, results_url: str) -> Dict[str, Any]: + """Open a list of results, wait for its first video and open that (its player is started + as on any video page reached).""" + self.open(results_url) + while True: + video = self._eval(_FIRST_VIDEO) + if video: + return self.open(video) + if self.deadline - time.monotonic() < 0.3: + raise BrowserError("no video came up for those words") + time.sleep(0.25) + def _press_play(self) -> None: """On a page just reached whose main player did not start: look again and press play.""" try: @@ -273,7 +295,7 @@ MAX_TABS = 5 _lock = threading.Lock() # Navigations that are safe to send again on a fresh connection; a click, typing or back may have # taken effect before the connection broke, so they are never repeated. -_REPEATABLE = ("open", "read") +_REPEATABLE = ("open", "read", "play") def _action(raw: Any) -> str: @@ -359,7 +381,10 @@ class BotBrowserTool(Tool): "Open and use a website in Jarvis's own web browser: search on a site, click, type into " "a box, read the page. " "Actions: open an address, read the current page, click a link or button by its label, " - "type into a search box or field (optionally pressing Enter), or go back. To search on a " + "type into a search box or field (optionally pressing Enter), go back, or play. To play a " + "song or video, call play once with the words as query, exactly as the user said them and " + "in their language (never translated): it searches YouTube, opens the first video and " + "starts it, and the page that comes back says whether it truly plays. To search on a " "site, call open with the site's address and the words as query, in plain text exactly as " "the user said them (never put the words into an address yourself); only the page that " "comes back shows whether the search ran. Use it when " @@ -375,11 +400,12 @@ class BotBrowserTool(Tool): "type": "object", "properties": { "action": {"type": "string", "enum": list(_ACTIONS), - "description": "open, read, click, type or back"}, + "description": "open, read, click, type, back or play"}, "url": {"type": "string", "description": "for open: the web address"}, "query": {"type": "string", "description": "for open: words to search for on that site, as " "plain text (never percent-encoded); they go into the address's empty search " - "parameter (e.g. ...?search_query=) or into the site's search box"}, + "parameter (e.g. ...?search_query=) or into the site's search box. For play: " + "the song or video to play, as the user said it"}, "target": {"type": "string", "description": "for click: the link or button label, or [n] from the list"}, "text": {"type": "string", "description": "for type: what to type"}, "field": {"type": "string", "description": "for type: the field's label or placeholder (optional)"}, @@ -410,13 +436,25 @@ class BotBrowserTool(Tool): call = lambda s: (s.open(url), s.type(words, None, True))[1] else: call = lambda s: s.open(url) + elif action == "play": + words = str(args.get("query") or args.get("text") or "").strip() + page = str(args.get("url") or "").strip() if not words else "" + # An address to play (given as url, or as the words): that page opens and its player starts. + address = _web_url(words or page) if re.match(r"^(https?://|www\.)", words or page, re.I) else None + if address: + call = lambda s: s.open(address) + elif not words: + return _fail("play needs the words of the song or video to play (query)") + else: + results = PLAY_SEARCH + urllib.parse.quote_plus(words) + call = lambda s: s.play(results) elif action == "click": target = str(args.get("target") or "").strip() if not target: return _fail("click needs the label of a link or button") call = lambda s: s.click(target) elif action == "type": - text = str(args.get("text") or "") + text = str(args.get("text") or args.get("query") or "") # models also write query if not text: return _fail("type needs the text to type") call = lambda s: s.type(text, args.get("field") or None, bool(args.get("submit"))) @@ -425,7 +463,7 @@ class BotBrowserTool(Tool): else: call = lambda s: s.read() who = str(getattr(context, "user_id", "") or "") - deadline = time.monotonic() + PAGE_WAIT_S # one deadline for connecting, the action and its page + deadline = time.monotonic() + (PLAY_WAIT_S if action == "play" else PAGE_WAIT_S) # one deadline for connecting, the action and its page t0 = time.perf_counter() with _lock: for attempt in (1, 2): diff --git a/src/jarvis/tools/builtin/bot_browser.spec.md b/src/jarvis/tools/builtin/bot_browser.spec.md index 0fb2dea..58ecd95 100644 --- a/src/jarvis/tools/builtin/bot_browser.spec.md +++ b/src/jarvis/tools/builtin/bot_browser.spec.md @@ -17,7 +17,7 @@ read it, click a link or button, type into a field, go back. Product rules: `doc `https://`); `file:`, `chrome:`, `data:`, `javascript:` and the rest are refused. The browser's own network rules (public addresses only) are the app's proxy's job. - **Raw data, no LLM call.** The tool returns plain text for the reply model. -- **Bounded.** One deadline of 15 s covers an action: connecting, the action and waiting for the +- **Bounded.** One deadline of 15 s (25 s for `play`) covers an action: connecting, the action and waiting for the page; events arriving in between do not extend it. Text is cut at 6,000 characters and the list of links and buttons at 40. - **A window per person.** Each asker (`ToolContext.user_id`) has a page and connection of their @@ -29,8 +29,8 @@ read it, click a link or button, type into a field, go back. Product rules: `doc - **One tab, also for "new tab" links.** Before a click or typing, links, forms and `` lose their `target` and `window.open` loads the address in the same tab, so what opens is what the tool reads next. -- **Never twice.** When the connection breaks during an action, `open` and `read` are tried once - more on a fresh connection (a fresh tab); `click`, `type` and `back` are not, because they may +- **Never twice.** When the connection breaks during an action, `open`, `read` and `play` (which + only opens pages) are tried once more on a fresh connection (a fresh tab); `click`, `type` and `back` are not, because they may already have taken effect: the tool says it may or may not have happened. - **The fence holds.** `<<<` and `>>>` in anything read from a page become `‹‹‹` and `›››`, so page text cannot close or reopen the fence. @@ -45,8 +45,9 @@ One tool, `action` picks the operation (read as its first known word, so `open_p | `open` | `url` (`query`) | opens the address in the bot browser's tab; with `query` (words to search for, plain text) they go, encoded by the tool (`urlencode`), into the address's last empty search parameter (`...?search_query=`), else the address opens and they are typed into its search box and sent (as `type` with `submit`) | | `read` | | reads the current page again (with a `url`, opens it) | | `click` | `target` | clicks the link or button whose label is `target` (an exact label first, else the one containing it; `[n]` from the last list also works) | -| `type` | `text` (`field`, `submit`) | types into the field whose label, placeholder or name contains `field` (else the focused field, else the first search or text box); `submit` presses Enter | +| `type` | `text` or `query` (`field`, `submit`) | types into the field whose label, placeholder or name contains `field` (else the focused field, else the first search or text box); `submit` presses Enter | | `back` | | goes back one page | +| `play` | `query` (or `url`) | plays a song or video: searches YouTube for the words (`.../results?search_query=`, encoded by the tool), waits for the first video in the results (a visible `/watch?v=` link, not one inside an ad), opens it by its plain address (`/watch?v=`, without the playlist the link may carry), and its player starts as on any video page opened (below); an address given (as `url`, or as words that start with `http(s)://` or `www.`) opens that page instead. No video in the results before the deadline is an error. Its deadline is 25 s (two pages and the start) | Every action answers with the page after it: title, address, the state of its sound and video (`Media: playing 1:15 of 3:45; paused 0:00`, each `