Compare commits

..

2 Commits

Author SHA1 Message Date
javis-bot
6a9b66bdcd chore(app): version 0.1.20
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 17:26:56 +09:00
javis-bot
cda8db7fce fix: play on YouTube in one step, Korea as the default location, searches in the user's language
- botBrowser `play`: searches YouTube for the words as said, opens the first video (not an ad,
  without the playlist) and starts it, in one call (an address given opens that page); `type`
  also takes `query`. Live 0.1.19: the model stopped on YouTube's home page or the results, a
  retry was refused as a third call, and the answer still said the video played.
- `brain.location` (기본 위치, default 대한민국) goes to the brain as `location_name`: the time line
  says it without GeoIP (no database on the game PC), and getWeather without a place uses it.
- Planner and webSearch keep search words in the user's language (never translated into
  English), so recommendations fit where the user is.

Evals (qwen3:8b, 3 runs): play ending on a playing video found with the words as said 0/6 -> 5/6;
Korean searches for Korean requests 6/9 -> 8/9.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 17:26:30 +09:00
24 changed files with 315 additions and 28 deletions

2
app/Cargo.lock generated
View File

@@ -1520,7 +1520,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
[[package]]
name = "jarvis-app"
version = "0.1.19"
version = "0.1.20"
dependencies = [
"async-trait",
"base64 0.22.1",

View File

@@ -1,6 +1,6 @@
[package]
name = "jarvis-app"
version = "0.1.19"
version = "0.1.20"
edition = "2021"
rust-version = "1.85"
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"

4
app/shell/Cargo.lock generated
View File

@@ -2637,7 +2637,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
[[package]]
name = "jarvis-app"
version = "0.1.19"
version = "0.1.20"
dependencies = [
"async-trait",
"base64 0.22.1",
@@ -2666,7 +2666,7 @@ dependencies = [
[[package]]
name = "jarvis-shell"
version = "0.1.19"
version = "0.1.20"
dependencies = [
"jarvis-app",
"serde_json",

View File

@@ -1,6 +1,6 @@
[package]
name = "jarvis-shell"
version = "0.1.19"
version = "0.1.20"
edition = "2021"
rust-version = "1.85"
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"

View File

@@ -662,10 +662,14 @@ start it first, so nothing answers before the brain can, and stop it last.
- **A server the app does not start**: with `llm.url` (an OpenAI-compatible `/v1` address, in the
settings file only) the app starts nothing; that server must list a model within 30 seconds.
- **Settings**: `llm.context` (default 32768 tokens per request, 4096-262144) and `llm.port`
(default 8080) on the model screen; `llm.url` (default none).
(default 8080) on the model screen; `llm.url` (default none). `brain.location` (기본 위치, on the
model screen, default `대한민국`, up to 100 characters, empty: the brain detects it): where the
people Jarvis answers are, for the time line, weather without a place and searches
(`src/jarvis/utils/location.spec.md`).
- **The brain's settings**: on every start the app writes `config/brain.json` (it owns the file):
`ollama_base_url` the LLM's `/v1`, `ollama_chat_model` the listed id, `db_path`
`memory/jarvis.db`, `disabled_tools` and `mcps` (above, "Features and MCP servers"); the worker gets it as `JARVIS_CONFIG_PATH`. Loading it writes nothing to the
`memory/jarvis.db`, `disabled_tools` and `mcps` (above, "Features and MCP servers"),
`location_name` (`brain.location`, null when empty); the worker gets it as `JARVIS_CONFIG_PATH`. Loading it writes nothing to the
home folder.
## Chat channel names and topics (`app/src/channel_sync.rs`)

View File

@@ -226,8 +226,14 @@ pub struct Settings {
/// The window's close button (X) hides it to the tray instead of quitting (`app.closeButton`:
/// "tray" or "quit", default "quit"; admin.spec.md, "Window and Windows build").
pub close_to_tray: bool,
/// Where the people Jarvis answers are, by default (`brain.location`, default "대한민국"): the
/// brain's location for the time line, weather without a place and searches (admin.spec.md,
/// "LLM"); None (an empty value) leaves it to the brain's own detection.
pub location: Option<String>,
}
pub const DEFAULT_LOCATION: &str = "대한민국";
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SettingsError(pub String);
@@ -279,6 +285,7 @@ impl Default for Settings {
type_text: Level::Ask, run: Level::Ask,
screen_exclude: ["1Password", "Bitwarden", "KeePass", "KeePassXC"].map(String::from).into() },
close_to_tray: false,
location: Some(DEFAULT_LOCATION.into()),
}
}
}
@@ -632,6 +639,13 @@ pub fn parse_settings(raw: &Value) -> Result<Settings, SettingsError> {
"quit" => false,
_ => return err("app.closeButton must be tray or quit"),
},
location: {
let l = get_str(section(raw, "brain")?, "location", "brain", DEFAULT_LOCATION)?.trim().to_owned();
if l.chars().count() > 100 {
return err("brain.location must be at most 100 characters");
}
(!l.is_empty()).then_some(l)
},
})
}
@@ -1006,6 +1020,12 @@ mod tests {
#[test]
fn the_close_button_choice_is_read_and_checked() {
assert!(!parse_settings(&json!({})).unwrap().close_to_tray, "closing quits by default");
assert_eq!(parse_settings(&json!({})).unwrap().location.as_deref(), Some(DEFAULT_LOCATION));
assert_eq!(parse_settings(&json!({"brain": {"location": " 부산 "}})).unwrap().location.as_deref(), Some("부산"));
assert_eq!(parse_settings(&json!({"brain": {"location": " "}})).unwrap().location, None, "empty: the brain detects it");
for bad in [json!({"brain": {"location": 1}}), json!({"brain": {"location": "가".repeat(101)}}), json!({"brain": 1})] {
assert!(parse_settings(&bad).unwrap_err().0.contains("brain"), "{bad}");
}
assert!(parse_settings(&json!({"app": {"closeButton": "tray"}})).unwrap().close_to_tray);
assert!(!parse_settings(&json!({"app": {"closeButton": "quit"}})).unwrap().close_to_tray);
for bad in [json!({"app": {"closeButton": "hide"}}), json!({"app": {"closeButton": 1}}), json!({"app": 1})] {

View File

@@ -128,6 +128,8 @@ pub fn fields() -> Vec<Field> {
f("voice.tts.speed", Model, Number { min: 0.5, max: 2.0 }, json!(1.5), "말하기 속도(1.0 보통, 기본 1.5, 저장하면 바로 적용)"),
// The LLM (admin.spec.md, "LLM"); `llm.url` (a server the app does not start) is file-only.
f("llm.context", Model, Int { min: 4096, max: 262144 }, json!(32768), "대화 모델이 한 번에 기억하는 양(토큰)"),
f("brain.location", Model, Text { max: 100, non_empty: false }, json!(crate::settings::DEFAULT_LOCATION),
"기본 위치(날씨, 검색, 추천의 기준, 비우면 자동 감지)"),
f("llm.port", Model, Int { min: 1024, max: 65535 }, json!(8080), "대화 모델 포트"),
// The brain's features (admin.spec.md, "Features and MCP servers").
f("features.weather", Features, Bool, json!(true), "날씨"),

View File

@@ -250,6 +250,7 @@ pub fn write_brain_config(data: &Path, base_url: &str, model: &str, settings: &c
"db_path": data.join("memory").join("jarvis.db").to_string_lossy(),
"disabled_tools": crate::features::disabled_tools(settings),
"mcps": crate::features::brain_mcps(&settings.mcp_servers),
"location_name": settings.location,
"_config_version": 1,
});
let mut text = serde_json::to_string_pretty(&cfg).unwrap();
@@ -716,6 +717,11 @@ pub(crate) mod tests {
assert!(Path::new(v["db_path"].as_str().unwrap()).starts_with(&l.data));
assert_eq!(v["disabled_tools"], json!(crate::features::disabled_tools(&s)));
assert_eq!(v["mcps"], crate::features::brain_mcps(&s.mcp_servers));
assert_eq!(v["location_name"], crate::settings::DEFAULT_LOCATION, "the people are in Korea by default");
let none = parse_settings(&json!({"brain": {"location": ""}})).unwrap();
let f = write_brain_config(&l.data, "http://127.0.0.1:8080/v1", "m", &none).unwrap();
let v: Value = serde_json::from_str(&std::fs::read_to_string(f).unwrap()).unwrap();
assert_eq!(v["location_name"], Value::Null);
}
#[cfg(unix)]

View File

@@ -164,7 +164,7 @@ Every distinct LLM call in Jarvis, what feeds it, what consumes it, and how it i
- **Trigger**: once per reply, **after the tool router and before memory search**. **v2 brain** (`intent_judge=True`): runs only on turns where the router picked a tool (not for the sentinels `stop` or `toolSearchTool` alone; a router-picked `chatChannels` counts), so a turn without tools has one LLM call before its answer. Skipped when `cfg.planner_enabled = False`, when the query is shorter than `MIN_QUERY_CHARS` (4), or when no model / base URL is available.
- **Model / gating**: resolution chain `planner_model (override) → ollama_chat_model`. The planner tracks the chat model so upgrading the chat model (via setup wizard or config) automatically upgrades plan quality.
- **Inputs**: user query, dialogue context, the live time/location line (`LIVE CONTEXT:`, so a location-dependent ask plans `searchMemory topic='where the user lives'` only when the location is unknown or disabled), **router-narrowed** tool catalogue (names + one-line descriptions + argument names, built by `planner_tool_catalog()`) — not the full 30+ list. When the carry-over guard from #7 fires, the previous turn's failed tool name is unioned into this catalogue before the planner sees it, so the planner can plan a re-call without `toolSearchTool` round-tripping. **No** memory context — the planner decides *whether* memory is needed.
- **System prompt**: `_PROMPT_TEMPLATE` in `planner.py`. Teaches the `searchMemory topic='...'` directive for prior-conversation lookups, short imperative tool steps, angle-bracket entity placeholders, final synthesis step, same-language output, no numbering.
- **System prompt**: `_PROMPT_TEMPLATE` in `planner.py`. Teaches the `searchMemory topic='...'` directive for prior-conversation lookups, short imperative tool steps, angle-bracket entity placeholders, final synthesis step, same-language output (the words of search and play arguments too, repeated in the user message's closing line), no numbering.
- **Output**: list of plan steps (max `MAX_STEPS` = 5). Gates memory enrichment (#3 / #4) and augments the tool router (#7 — planner's picks are unioned in, not replacing). Single-step `["Reply to the user."]` plans are the planner's positive "no memory, no tools" signal. An empty list is fail-open — the engine reverts to running #3 unconditionally. Consumed further by the engine to build the `ACTION PLAN:` system-message block and drive the direct-exec loop (#13) for small models.
- **Limits**: `planner_timeout_sec` (6s). Fail-open → `[]`.

View File

@@ -18,6 +18,8 @@ Cases:
Run: EVAL_JUDGE_MODEL=qwen2.5:7b python -m pytest evals/test_bot_browser_tool.py -v
"""
import re
import pytest
from conftest import requires_judge_llm
@@ -88,6 +90,15 @@ class YoutubeSession(FakeSession):
self.url = "https://www.youtube.com/results?search_query=x"
return dict(self.RESULTS)
def play(self, results_url):
# The real session opens the first video and starts its player (tests/test_bot_browser.py).
import urllib.parse
self.calls.append(("play", results_url))
self.url = "https://www.youtube.com/watch?v=nP1xjFX86Po"
self.playing = True
words = urllib.parse.unquote_plus(results_url.split("search_query=")[-1])
return dict(self._page(self.url), title=f"{words} (Official MV) - YouTube", text=f"{words} (Official MV)")
def click(self, target):
self.calls.append(("click", target))
if "watch" in getattr(self, "url", ""):
@@ -148,17 +159,34 @@ class TestBotBrowserTool:
import urllib.parse
s, _ = _run(monkeypatch, mock_config, eval_db, eval_dialogue_memory, "유튜브에서 아이유 신곡 틀어 줘",
session=YoutubeSession())
words = [urllib.parse.unquote_plus(c[1]) for c in s.calls if c[0] == "open"] + \
words = [urllib.parse.unquote_plus(c[1]) for c in s.calls if c[0] in ("open", "play")] + \
[c[1] for c in s.calls if c[0] == "type"]
assert any("youtube" in urllib.parse.unquote_plus(c[1]).lower() for c in s.calls if c[0] == "open"), s.calls
assert any("youtube" in urllib.parse.unquote_plus(c[1]).lower() for c in s.calls if c[0] in ("open", "play")), s.calls
assert any("아이유" in w or "iu" in w.lower() for w in words), s.calls
assert not any("아이우" in w for w in words), s.calls
@pytest.mark.parametrize("query,said", [("유튜브에서 아이유 신곡 틀어 줘", ("아이유", "iu")),
("자비스, 유튜브에서 뉴진스 하입보이 틀어 줘", ("뉴진스", "newjeans"))])
def test_playing_a_song_ends_on_a_playing_video_found_with_the_words_as_said(
self, monkeypatch, mock_config, eval_db, eval_dialogue_memory, query, said):
# Live 2026-11-16 (0.1.19): the model stopped on the results page or on YouTube's home page
# (a third browser call was refused), searched "NewJeans Hypothesis" in English, and still
# said the video played.
import urllib.parse
s, reply = _run(monkeypatch, mock_config, eval_db, eval_dialogue_memory, query, session=YoutubeSession())
assert s.playing and "watch" in s.url, f"{s.calls} reply={reply!r}"
searched = [urllib.parse.unquote_plus(c[1]) for c in s.calls if c[0] in ("open", "play")] + \
[c[1] for c in s.calls if c[0] == "type"]
# The words in the language they were said in (a Korean request searches in Korean, so the
# results are the ones people there get), naming who was asked for.
assert any(re.search(r"[가-힣]", w) and any(n in w.lower() for n in said) for w in searched), s.calls
def test_a_video_page_that_stays_paused_is_played_or_reported_as_not_playing(
self, monkeypatch, mock_config, eval_db, eval_dialogue_memory):
s, reply = _run(monkeypatch, mock_config, eval_db, eval_dialogue_memory,
"브라우저로 https://www.youtube.com/watch?v=nP1xjFX86Po 열어서 영상 재생해 줘", session=YoutubeSession())
assert any(c[0] == "open" and "watch" in c[1] for c in s.calls), s.calls
# It reached the video (opened or played by its address or its ID).
assert "watch" in getattr(s, "url", ""), s.calls
# Either it pressed play, or it told the user honestly that the video did not play (a
# negation in the Korean reply); never "it plays" over a paused video.
assert s.playing or any(n in (reply or "") for n in ("않", "못")), f"{s.calls} reply={reply!r}"

View File

@@ -0,0 +1,54 @@
"""
Eval: a Korean request searches the web in Korean, so the results are the ones people in Korea get.
Live 2026-11-16 (0.1.19): "요즘 볼 만한 넷플릭스 드라마 세 개만 추천해 줘" searched "best Netflix dramas to
watch now 2026" and recommended American shows; "엔비디아 rtx 5050 그래픽카드 가격 검색해 줘" searched in
English and answered with an Amazon US price. The reply engine runs as the v2 brain runs it (compact
prompt, Korean default, the place set in the settings: brain.location, default 대한민국); webSearch is
a fake that records its query. Checked by script: the query has Hangul.
Run: EVAL_JUDGE_MODEL=qwen3:8b python -m pytest evals/test_search_in_users_language.py -v
"""
import re
from unittest.mock import patch
import pytest
from conftest import requires_judge_llm
from helpers import ToolCallCapture, create_mock_tool_run
CASES = [
("netflix", "요즘 볼 만한 넷플릭스 드라마 세 개만 추천해 줘"),
("price", "엔비디아 RTX 5050 그래픽카드 가격 검색해 줘"),
("restaurants", "이번 주말에 갈 만한 맛집 좀 찾아 줘"),
]
@pytest.mark.eval
@requires_judge_llm
class TestSearchInUsersLanguage:
@pytest.mark.parametrize("case", CASES, ids=[c[0] for c in CASES])
def test_a_korean_request_searches_in_korean(self, mock_config, eval_db, eval_dialogue_memory,
monkeypatch, case):
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
from jarvis.memory.history import HistoryContext
from jarvis.reply.engine import run_reply_engine
cid, query = case
monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False)
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
mock_config.location_name = "대한민국"
capture = ToolCallCapture()
result = "Search results: (fake) 1. 결과 하나 2. 결과 둘"
with patch("jarvis.reply.engine.run_tool_with_retries",
side_effect=create_mock_tool_run(capture, {"webSearch": result})):
reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
dialogue_memory=eval_dialogue_memory, history=HistoryContext(),
legacy_memory=False, default_language="Korean", compact_prompt=True)
searches = [str(c["args"].get("search_query") or c["args"].get("query") or "")
for c in capture.calls if c["name"] == "webSearch"]
print(f"\n search language [{JUDGE_MODEL}] {cid}: searches={searches} reply={(reply or '')[:120]!r}")
assert searches, f"[{cid}] no web search: {capture.tool_names()}"
assert all(re.search(r"[가-힣]", q) for q in searches), f"[{cid}] searched in another language: {searches}"

View File

@@ -238,6 +238,8 @@ class Settings:
location_ip_address: str | None
location_auto_detect: bool
location_cgnat_resolve_public_ip: bool
# A place the user set (e.g. "대한민국"): used instead of detection when set (location.spec.md).
location_name: str | None
# Web Search
web_search_enabled: bool
@@ -551,6 +553,7 @@ def get_default_config() -> Dict[str, Any]:
# When behind CGNAT (100.64.0.0/10), attempt a privacy-light external DNS query to discover true public IP.
# Uses a single OpenDNS resolver lookup of myip.opendns.com over DNS (no HTTP services). Disable to avoid any external request.
"location_cgnat_resolve_public_ip": True,
"location_name": None,
# Web Search
"web_search_enabled": True,
@@ -762,6 +765,7 @@ def load_settings() -> Settings:
location_ip_address = None if location_ip_address_val in (None, "", "null") else str(location_ip_address_val)
location_auto_detect = bool(merged.get("location_auto_detect", True))
location_cgnat_resolve_public_ip = bool(merged.get("location_cgnat_resolve_public_ip", True))
location_name = str(merged.get("location_name") or "").strip() or None
web_search_enabled = bool(merged.get("web_search_enabled", True))
brave_search_api_key = str(merged.get("brave_search_api_key", "") or "").strip()
wikipedia_fallback_enabled = bool(merged.get("wikipedia_fallback_enabled", True))
@@ -902,6 +906,7 @@ def load_settings() -> Settings:
location_ip_address=location_ip_address,
location_auto_detect=location_auto_detect,
location_cgnat_resolve_public_ip=location_cgnat_resolve_public_ip,
location_name=location_name,
# Web Search
web_search_enabled=web_search_enabled,

View File

@@ -761,6 +761,9 @@ def _live_time_location_string(cfg) -> str:
tz_name: Optional[str] = None
if not getattr(cfg, 'location_enabled', True):
location_context = "Location: Disabled"
elif isinstance(getattr(cfg, 'location_name', None), str) and cfg.location_name.strip():
# The place the user set wins over detection (location.spec.md); local time is the PC's.
location_context = f"Location: {cfg.location_name.strip()} (where the user is)"
else:
location_context, tz_name = get_location_context_with_timezone(
config_ip=getattr(cfg, 'location_ip_address', None),

View File

@@ -232,7 +232,10 @@ _PROMPT_TEMPLATE = (
"10. Maximum {max_steps} steps. Do not number them — one step per line.\n"
"11. Output ONLY the steps, no preamble, no trailing commentary, no "
"JSON fences, no explanations.\n"
"12. Write the steps in the same language the user wrote the query in.\n"
"12. Write the steps in the same language the user wrote the query in. "
"The words of every search or play argument stay in the user's "
"language too, never translated into English, because a search in the "
"user's language finds what people where they are get.\n"
)
@@ -279,7 +282,8 @@ def _build_user_message(
else:
parts.append("DIALOGUE CONTEXT: (empty)")
parts.append(f"USER QUERY: {query.strip()}")
parts.append("\nEmit the plan now, one step per line, no numbering.")
parts.append("\nEmit the plan now, one step per line, no numbering, written in the language of "
"the USER QUERY (the words of searches too).")
return "\n\n".join(parts)

View File

@@ -108,7 +108,14 @@ The planner prompt instructs the model to emit:
`webSearch query='his most famous songs'`.
- A final synthesis/reply step when any `searchMemory` or tool step
was planned.
- Steps in the same language the user wrote the query in.
- Steps in the same language the user wrote the query in, the words of
search and play arguments too (never translated into English: a search
in the user's language finds what people where they are get); the
closing line of the user message repeats it, which a small model follows
better than the rule alone. Eval `evals/test_search_in_users_language.py`
(qwen3:8b, 3 runs of 3 Korean requests): Korean searches 6/9 before
("요즘 볼 만한 넷플릭스 드라마" searched "best netflix dramas 2026" 3/3), 8/9
after (that one 2/3).
### Parsing and hygiene

View File

@@ -18,8 +18,12 @@ from ...debug import debug_log
FENCE_START = "<<<BEGIN UNTRUSTED WEB PAGE>>>"
FENCE_END = "<<<END UNTRUSTED WEB PAGE>>>"
_ACTIONS = ("open", "read", "click", "type", "back")
_ACTIONS = ("open", "read", "click", "type", "back", "play")
PAGE_WAIT_S = 15.0
# play: two pages (the results and the video) and the wait for the video to start.
PLAY_WAIT_S = 25.0
# Where play searches: YouTube's results for the words (encoded by the tool).
PLAY_SEARCH = "https://www.youtube.com/results?search_query="
TEXT_CHARS = 6000
MAX_ITEMS = 40
CDP_ENV = "JARVIS_BROWSER_CDP"
@@ -58,6 +62,12 @@ _PLAY = r"""
(() => { const v = [...document.querySelectorAll('video')].filter(m => (m.currentSrc || m.src) && m.paused
&& !m.ended && !m.error && m.currentTime === 0 && m.getBoundingClientRect().width >= innerWidth * 0.5);
if (!v.length) return false; v[0].play().catch(() => {}); return true; })()"""
# The first video in a list of results (a visible `/watch?v=` link, not an ad), as its plain address.
_FIRST_VIDEO = "(() => {" + _HELPERS + r"""
const a = [...document.querySelectorAll('a[href*="/watch?v="]')].find(e => vis(e)
&& !e.closest('ytd-ad-slot-renderer,ytd-promoted-video-renderer,ytd-in-feed-ad-layout-renderer,[is-ad]'));
if (!a) return null; const u = new URL(a.getAttribute('href'), location.href);
return u.origin + u.pathname + '?v=' + encodeURIComponent(u.searchParams.get('v')); })()"""
_CLICK = "(() => {" + _HELPERS + r"""
return ((t) => { const m = /^\s*\[(\d+)\]\s*$/.exec(t); let el = null;
if (m) el = (window.__jvItems || [])[Number(m[1]) - 1];
@@ -239,6 +249,18 @@ class BrowserSession:
self._press_play()
return self.read()
def play(self, results_url: str) -> Dict[str, Any]:
"""Open a list of results, wait for its first video and open that (its player is started
as on any video page reached)."""
self.open(results_url)
while True:
video = self._eval(_FIRST_VIDEO)
if video:
return self.open(video)
if self.deadline - time.monotonic() < 0.3:
raise BrowserError("no video came up for those words")
time.sleep(0.25)
def _press_play(self) -> None:
"""On a page just reached whose main player did not start: look again and press play."""
try:
@@ -273,7 +295,7 @@ MAX_TABS = 5
_lock = threading.Lock()
# Navigations that are safe to send again on a fresh connection; a click, typing or back may have
# taken effect before the connection broke, so they are never repeated.
_REPEATABLE = ("open", "read")
_REPEATABLE = ("open", "read", "play")
def _action(raw: Any) -> str:
@@ -359,7 +381,10 @@ class BotBrowserTool(Tool):
"Open and use a website in Jarvis's own web browser: search on a site, click, type into "
"a box, read the page. "
"Actions: open an address, read the current page, click a link or button by its label, "
"type into a search box or field (optionally pressing Enter), or go back. To search on a "
"type into a search box or field (optionally pressing Enter), go back, or play. To play a "
"song or video, call play once with the words as query, exactly as the user said them and "
"in their language (never translated): it searches YouTube, opens the first video and "
"starts it, and the page that comes back says whether it truly plays. To search on a "
"site, call open with the site's address and the words as query, in plain text exactly as "
"the user said them (never put the words into an address yourself); only the page that "
"comes back shows whether the search ran. Use it when "
@@ -375,11 +400,12 @@ class BotBrowserTool(Tool):
"type": "object",
"properties": {
"action": {"type": "string", "enum": list(_ACTIONS),
"description": "open, read, click, type or back"},
"description": "open, read, click, type, back or play"},
"url": {"type": "string", "description": "for open: the web address"},
"query": {"type": "string", "description": "for open: words to search for on that site, as "
"plain text (never percent-encoded); they go into the address's empty search "
"parameter (e.g. ...?search_query=) or into the site's search box"},
"parameter (e.g. ...?search_query=) or into the site's search box. For play: "
"the song or video to play, as the user said it"},
"target": {"type": "string", "description": "for click: the link or button label, or [n] from the list"},
"text": {"type": "string", "description": "for type: what to type"},
"field": {"type": "string", "description": "for type: the field's label or placeholder (optional)"},
@@ -410,13 +436,25 @@ class BotBrowserTool(Tool):
call = lambda s: (s.open(url), s.type(words, None, True))[1]
else:
call = lambda s: s.open(url)
elif action == "play":
words = str(args.get("query") or args.get("text") or "").strip()
page = str(args.get("url") or "").strip() if not words else ""
# An address to play (given as url, or as the words): that page opens and its player starts.
address = _web_url(words or page) if re.match(r"^(https?://|www\.)", words or page, re.I) else None
if address:
call = lambda s: s.open(address)
elif not words:
return _fail("play needs the words of the song or video to play (query)")
else:
results = PLAY_SEARCH + urllib.parse.quote_plus(words)
call = lambda s: s.play(results)
elif action == "click":
target = str(args.get("target") or "").strip()
if not target:
return _fail("click needs the label of a link or button")
call = lambda s: s.click(target)
elif action == "type":
text = str(args.get("text") or "")
text = str(args.get("text") or args.get("query") or "") # models also write query
if not text:
return _fail("type needs the text to type")
call = lambda s: s.type(text, args.get("field") or None, bool(args.get("submit")))
@@ -425,7 +463,7 @@ class BotBrowserTool(Tool):
else:
call = lambda s: s.read()
who = str(getattr(context, "user_id", "") or "")
deadline = time.monotonic() + PAGE_WAIT_S # one deadline for connecting, the action and its page
deadline = time.monotonic() + (PLAY_WAIT_S if action == "play" else PAGE_WAIT_S) # one deadline for connecting, the action and its page
t0 = time.perf_counter()
with _lock:
for attempt in (1, 2):

View File

@@ -17,7 +17,7 @@ read it, click a link or button, type into a field, go back. Product rules: `doc
`https://`); `file:`, `chrome:`, `data:`, `javascript:` and the rest are refused. The browser's
own network rules (public addresses only) are the app's proxy's job.
- **Raw data, no LLM call.** The tool returns plain text for the reply model.
- **Bounded.** One deadline of 15 s covers an action: connecting, the action and waiting for the
- **Bounded.** One deadline of 15 s (25 s for `play`) covers an action: connecting, the action and waiting for the
page; events arriving in between do not extend it. Text is cut at 6,000 characters and the list
of links and buttons at 40.
- **A window per person.** Each asker (`ToolContext.user_id`) has a page and connection of their
@@ -29,8 +29,8 @@ read it, click a link or button, type into a field, go back. Product rules: `doc
- **One tab, also for "new tab" links.** Before a click or typing, links, forms and `<base>` lose
their `target` and `window.open` loads the address in the same tab, so what opens is what the
tool reads next.
- **Never twice.** When the connection breaks during an action, `open` and `read` are tried once
more on a fresh connection (a fresh tab); `click`, `type` and `back` are not, because they may
- **Never twice.** When the connection breaks during an action, `open`, `read` and `play` (which
only opens pages) are tried once more on a fresh connection (a fresh tab); `click`, `type` and `back` are not, because they may
already have taken effect: the tool says it may or may not have happened.
- **The fence holds.** `<<<` and `>>>` in anything read from a page become `‹‹‹` and `›››`, so page
text cannot close or reopen the fence.
@@ -45,8 +45,9 @@ One tool, `action` picks the operation (read as its first known word, so `open_p
| `open` | `url` (`query`) | opens the address in the bot browser's tab; with `query` (words to search for, plain text) they go, encoded by the tool (`urlencode`), into the address's last empty search parameter (`...?search_query=`), else the address opens and they are typed into its search box and sent (as `type` with `submit`) |
| `read` | | reads the current page again (with a `url`, opens it) |
| `click` | `target` | clicks the link or button whose label is `target` (an exact label first, else the one containing it; `[n]` from the last list also works) |
| `type` | `text` (`field`, `submit`) | types into the field whose label, placeholder or name contains `field` (else the focused field, else the first search or text box); `submit` presses Enter |
| `type` | `text` or `query` (`field`, `submit`) | types into the field whose label, placeholder or name contains `field` (else the focused field, else the first search or text box); `submit` presses Enter |
| `back` | | goes back one page |
| `play` | `query` (or `url`) | plays a song or video: searches YouTube for the words (`.../results?search_query=`, encoded by the tool), waits for the first video in the results (a visible `/watch?v=` link, not one inside an ad), opens it by its plain address (`/watch?v=<id>`, without the playlist the link may carry), and its player starts as on any video page opened (below); an address given (as `url`, or as words that start with `http(s)://` or `www.`) opens that page instead. No video in the results before the deadline is an error. Its deadline is 25 s (two pages and the start) |
Every action answers with the page after it: title, address, the state of its sound and video
(`Media: playing 1:15 of 3:45; paused 0:00`, each `<video>` and `<audio>` with a source, `(muted)`
@@ -55,7 +56,13 @@ When the page has a main player (a video at least half the window wide) and noth
note follows the fence: the video is not playing, do not say it plays; if it should, click its play
button and look again, and if it still does not play, tell the user it did not start. The
description tells the model to search a site with `open` and `query` in plain text, never putting the
words into an address itself (live 2026-11-16: 아이유 hand-encoded into 아이우). Eval
words into an address itself (live 2026-11-16: 아이유 hand-encoded into 아이우), and to play a song or
video with one `play` whose `query` is the words as the user said them, in their language (live
2026-11-16, 0.1.19: "아이유 신곡 틀어 줘" opened YouTube's home page, then a `type` with `query` instead of
`text` failed and its retry was refused as a third call of the tool in one answer; "뉴진스 하입보이"
searched "NewJeans Hypothesis" in English and stopped on the results; both answers said the video
played). Eval `evals/test_bot_browser_tool.py` (qwen3:8b, 3 runs of the two requests): ending on a
playing video found with the words as said 0/6 before, 5/6 after (6/6 played; once "IU new song"). Eval
`evals/test_bot_browser_tool.py` (qwen3:8b, 3 runs each): reaching YouTube with the right words 1/3
before, 3/3 after; a video page that stays paused reported as playing 1/3 before, never after (3/3
said it did not play).

View File

@@ -289,6 +289,13 @@ class WeatherTool(Tool):
# debug log, doesn't change behaviour downstream.
place_from_fallback = False
set_place = getattr(getattr(context, "cfg", None), "location_name", None)
set_place = set_place.strip() if isinstance(set_place, str) else ""
if not location_str and set_place:
# No place asked for: the place the user set in the settings (location.spec.md).
location_str = set_place
debug_log(" 📍 using the location set in the settings", "tools")
if not location_str:
# No location provided - try auto-detected coordinates first.
user_loc = self._get_user_location(context)

View File

@@ -570,7 +570,7 @@ class WebSearchTool(Tool):
return {
"type": "object",
"properties": {
"search_query": {"type": "string", "description": "A self-contained search query with entity names resolved from conversation history (not a literal echo of the user's utterance). Prefer a compact keyword phrase over a conversational sentence — e.g. 'Harry Styles most famous songs', not 'what are his most famous songs'."}
"search_query": {"type": "string", "description": "A self-contained search query with entity names resolved from conversation history (not a literal echo of the user's utterance), in the language the user spoke (never translated into English: results then fit where the user is). Prefer a compact keyword phrase over a conversational sentence — e.g. 'Harry Styles most famous songs', not 'what are his most famous songs'."}
},
"required": ["search_query"]
}

View File

@@ -7,7 +7,10 @@ memory.
The query comes in `search_query`; `query` is read when `search_query` is missing or empty (the
planner writes steps as `webSearch query='...'` and models copy it; live 2026-11-16 both searches of
an answer failed on it).
an answer failed on it). Its description asks for the query in the language the user spoke, never
translated into English, so the results fit where the user is (live 2026-11-16: a Korean request for
Netflix dramas searched in English and recommended American shows; eval
`evals/test_search_in_users_language.py`, with the planner rule in `planner.spec.md`).
### Real-time info routing (`STREAM_BROWSER`)

View File

@@ -17,6 +17,7 @@ This specification documents the location detection module (`src/jarvis/utils/lo
| `location_ip_address` | str \| null | `null` | Manually configured public IP. Takes precedence over auto-detection when set. |
| `location_cgnat_resolve_public_ip` | bool | `true` | When a CGNAT address (100.64.0.0/10) is detected, attempt a single DNS query to OpenDNS to resolve the true public IP. Disable to prevent any external DNS query. |
| `location_cache_minutes` | int | `60` | TTL for cached location lookups persisted to disk. |
| `location_name` | str \| null | `null` | A place the user set (v2: the app's `brain.location`, default `대한민국`). When set (and `location_enabled`), the reply engine's time line says `Location: <name> (where the user is)` without any detection, and `getWeather` without a place asked for uses it (a place the person names still wins). The local time is then the computer's. Live 2026-11-16: the game PC has no GeoLite2 database, so detection always failed ("Location: Unknown", weather asked for the city) and searches fitted no country. |
### IP Resolution Chain

View File

@@ -58,6 +58,10 @@ class FakeSession:
self.calls.append(("back",))
return self.page
def play(self, results_url):
self.calls.append(("play", results_url))
return self.page
@pytest.fixture
def fake(monkeypatch):
@@ -176,6 +180,25 @@ def test_actions_pass_their_arguments(fake):
assert not t.run({"action": "dance"}, ctx()).success
def test_play_searches_youtube_for_the_words_as_said_in_one_call(fake):
t = BotBrowserTool()
assert t.run({"action": "play", "query": "뉴진스 하입보이"}, ctx()).success
assert fake.calls == [("play", "https://www.youtube.com/results?search_query="
"%EB%89%B4%EC%A7%84%EC%8A%A4+%ED%95%98%EC%9E%85%EB%B3%B4%EC%9D%B4")]
assert not t.run({"action": "play"}, ctx()).success # nothing to play
# An address to play opens that page (whose player starts as on any video page opened).
fake.calls.clear()
t.run({"action": "play", "query": "https://www.youtube.com/watch?v=x"}, ctx())
t.run({"action": "play", "url": "www.youtube.com/watch?v=y"}, ctx())
assert fake.calls == [("open", "https://www.youtube.com/watch?v=x"), ("open", "https://www.youtube.com/watch?v=y")]
def test_typing_takes_the_words_also_as_query(fake):
# Live 2026-11-16: the model wrote query instead of text and the search never ran.
BotBrowserTool().run({"action": "type", "query": "아이유 신곡", "submit": True}, ctx())
assert fake.calls == [("type", "아이유 신곡", None, True)]
class Broken(FakeSession):
"""A session whose connection breaks during `breaks` (after the action may have gone out)."""
@@ -300,6 +323,15 @@ PAGES = {
"/results": "<html><head><title>Results</title></head><body><a href='/watch'>The song</a></body></html>",
"/watch": "<html><head><title>Watch</title></head><body>"
"<video src='/tone.wav' style='width:90vw;height:300px'></video></body></html>",
# Search results with an ad first, then a video (on YouTube the links carry ?v= and more).
"/yt": "<html><head><title>Results</title></head><body>"
"<div is-ad><a href='/watch?v=ad1'>An ad</a></div>"
"<a href='/watch?v=abc&list=RDabc&start_radio=1'>The song</a></body></html>",
# Results that show their first video only a moment after the page loaded.
"/slow": "<html><head><title>Results</title></head><body><script>setTimeout(() => {"
"const a = document.createElement('a'); a.href = '/watch?v=late'; a.textContent = 'Late song';"
"document.body.appendChild(a); }, 1500)</script></body></html>",
"/none": "<html><head><title>Results</title></head><body>No results</body></html>",
# A video page whose player cannot play (its file is missing).
"/broken": "<html><head><title>Broken</title></head><body>"
"<video src='/missing.mp4' style='width:90vw;height:300px'></video></body></html>",
@@ -425,6 +457,20 @@ def test_a_video_page_whose_player_stays_paused_gets_play_pressed_when_clicked_t
assert "not playing" in bot_browser._render(page)
def test_play_opens_the_first_video_of_the_results_not_an_ad_and_starts_it(chrome):
cdp, site, seen = chrome
s = BrowserSession(cdp, time.monotonic() + 25)
page = s.play(site + "/yt?search_query=x")
assert page["url"] == site + "/watch?v=abc", page["url"]
assert [m["playing"] for m in page["media"]] == [True], page
assert "/watch?v=ad1" not in seen
s.deadline = time.monotonic() + 25
assert s.play(site + "/slow")["url"] == site + "/watch?v=late"
s.deadline = time.monotonic() + 3
with pytest.raises(bot_browser.BrowserError):
s.play(site + "/none")
def test_links_that_would_open_a_new_tab_open_in_the_same_tab(chrome):
cdp, site, _ = chrome
s = BrowserSession(cdp, time.monotonic() + 15)

View File

@@ -111,3 +111,33 @@ def test_auto_detect_rejects_private_ip_from_opendns():
patch("jarvis.utils.location._resolve_public_ip_via_opendns", return_value="192.168.1.1"):
result = _get_external_ip_automatically()
assert result is None
def test_a_place_set_in_the_settings_is_the_location_and_detection_is_not_tried():
"""brain.location (default 대한민국) reaches the brain as location_name: the time line says it,
without GeoIP (which has no database on the game PC: "Location: Unknown" before)."""
from jarvis.reply import engine
with patch.object(engine, "get_location_context_with_timezone") as detect:
line = engine._live_time_location_string(types.SimpleNamespace(location_enabled=True, location_name="대한민국"))
assert "Location: 대한민국" in line and "Current local time" in line
detect.assert_not_called()
# Location turned off still wins; no place set means detection as before.
off = engine._live_time_location_string(types.SimpleNamespace(location_enabled=False, location_name="대한민국"))
assert "Location: Disabled" in off
with patch.object(engine, "get_location_context_with_timezone", return_value=("Location: Seoul", None)) as detect:
assert "Location: Seoul" in engine._live_time_location_string(types.SimpleNamespace(location_enabled=True, location_name=None))
detect.assert_called_once()
def test_the_settings_file_carries_the_place():
import json
import tempfile
from jarvis.config import load_settings
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f:
json.dump({"location_name": " 대한민국 "}, f)
with patch.dict("os.environ", {"JARVIS_CONFIG_PATH": f.name}):
assert load_settings().location_name == "대한민국"
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f:
json.dump({"location_name": None}, f)
with patch.dict("os.environ", {"JARVIS_CONFIG_PATH": f.name}):
assert load_settings().location_name is None

View File

@@ -253,6 +253,28 @@ class TestWeatherTool:
# Verify location detection was called (fallback was attempted)
mock_location.assert_called_once()
@patch('requests.get')
@patch('src.jarvis.tools.builtin.weather.get_location_info')
def test_without_a_place_the_place_set_in_the_settings_is_used(self, mock_location, mock_get):
"""Live 2026-11-16: detection failed on the game PC and "날씨 어때" got "tell me the city";
the place set in the settings (brain.location, default 대한민국) is the answer instead."""
self.context.cfg.location_name = "South Korea"
geo = Mock(raise_for_status=Mock())
geo.json.return_value = {"results": [{"latitude": 36.5, "longitude": 127.75, "name": "South Korea",
"country": "South Korea"}]}
weather = Mock(raise_for_status=Mock())
weather.json.return_value = {"current": {"temperature_2m": 12.0, "apparent_temperature": 12.0,
"weather_code": 0}}
mock_get.side_effect = [geo, weather]
result = self.tool.run({}, self.context)
assert result.success, result.reply_text
assert mock_get.call_args_list[0].kwargs["params"]["name"] == "South Korea"
mock_location.assert_not_called()
# A place the person asked about still wins over the set one.
mock_get.side_effect = [geo, weather]
self.tool.run({"location": "Busan"}, self.context)
assert mock_get.call_args_list[2].kwargs["params"]["name"] == "Busan"
@patch('src.jarvis.tools.builtin.weather.get_location_info')
def test_run_no_args_uses_fallback(self, mock_location):
"""Test weather with no arguments uses user's detected location as fallback."""