From 5d45d1d3bd030fbc361241817c38204d85ad4837 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Thu, 11 Jun 2026 01:17:50 +0900 Subject: [PATCH] feat(brain): route real-time search by live broadcast state STREAM_BROWSER becomes the broadcast *capability* master flag; the live screen-share state (new ToolContext.broadcasting, passed per turn by the bot) decides the backend: - master off -> broadcast disabled, always Gemini - master on + live on -> on-screen Chrome (visible on the stream) - master on + live off -> Gemini context.broadcasting is None outside the voice path (evals, text entry, older bot) and falls back to the master flag, so current behaviour is unchanged. This is the brain-side foundation; bot-side wiring (bridge passes broadcast state, auto-broadcast on voice join, voice on/off toggle) follows. Specs + docs/llm_contexts.md updated. Covered by tests/test_web_search_broadcast_routing.py. Co-Authored-By: Claude Opus 4.7 --- docs/llm_contexts.md | 2 +- docs/stream_browser_modes.md | 34 +++++++--- src/jarvis/tools/base.py | 8 +++ src/jarvis/tools/builtin/web_search.py | 24 +++++-- src/jarvis/tools/builtin/web_search.spec.md | 23 +++++-- tests/test_web_search_broadcast_routing.py | 75 +++++++++++++++++++++ 6 files changed, 144 insertions(+), 22 deletions(-) create mode 100644 tests/test_web_search_broadcast_routing.py diff --git a/docs/llm_contexts.md b/docs/llm_contexts.md index e55f803..8bf036c 100644 --- a/docs/llm_contexts.md +++ b/docs/llm_contexts.md @@ -171,7 +171,7 @@ Every distinct LLM call in Jarvis, what feeds it, what consumes it, and how it i - **Weather** ([src/jarvis/tools/builtin/weather.py](src/jarvis/tools/builtin/weather.py), ~line 60) — `ollama_chat_model`, parses location/time/unit from the query. - **Nutrition log_meal** ([src/jarvis/tools/builtin/nutrition/log_meal.py](src/jarvis/tools/builtin/nutrition/log_meal.py), lines 48 & 136) — `ollama_chat_model`, extracts nutrients, confirms logging. -- **Gemini real-time search** ([src/jarvis/tools/builtin/realtime_search.py](src/jarvis/tools/builtin/realtime_search.py)) — **external Gemini model**, NOT Ollama. Only on the `webSearch` route when `STREAM_BROWSER=false`; the sub-mode is `cfg.gemini_auth` (env `GEMINI_AUTH`, default `oauth`): +- **Gemini real-time search** ([src/jarvis/tools/builtin/realtime_search.py](src/jarvis/tools/builtin/realtime_search.py)) — **external Gemini model**, NOT Ollama. Used on the `webSearch` route whenever the on-screen Chrome path is NOT active: either `STREAM_BROWSER=false` (broadcast disabled) or `STREAM_BROWSER=true` with the live broadcast currently off (`context.broadcasting` False). Sub-mode is `cfg.gemini_auth` (env `GEMINI_AUTH`, default `oauth`): - `oauth` (default) `gemini_cli_search()` — shells out to the Gemini CLI (`gemini -p -o json --skip-trust`, default approval mode) authenticated by the user's Google-account login (`GEMINI_API_KEY`/`GOOGLE_API_KEY` stripped from the child env, `GOOGLE_GENAI_USE_GCA=true` set to select OAuth); model is whatever the CLI/account defaults to. Uses the CLI's built-in web-search grounding. Bounded by a 30s subprocess timeout. - `apikey` `gemini_search()` — one REST `generateContent` call (`gemini_model`, default `gemini-2.0-flash`) with the `google_search` grounding tool; keyed by `GEMINI_API_KEY`. Both return the fenced UNTRUSTED-WEB-EXTRACT envelope consumed by the main loop (#1). Fail-open: CLI missing / login expired / quota 429 / timeout / errors / missing key all fall through to the DDG cascade. The `STREAM_BROWSER=true` route (`browser_search()`) makes NO LLM call — it drives Chrome and scrapes Google results. diff --git a/docs/stream_browser_modes.md b/docs/stream_browser_modes.md index 82fc4fd..edbfa9d 100644 --- a/docs/stream_browser_modes.md +++ b/docs/stream_browser_modes.md @@ -1,16 +1,29 @@ # Real-time info modes (`STREAM_BROWSER`) The bot answers via the Python brain (`bridge/server.py` -> `src/jarvis`). Real-time -info is fetched by a tool the reply engine calls. `STREAM_BROWSER` selects HOW: +info is fetched by a tool the reply engine calls. -- **true** (default): drive the on-screen Chrome (CDP at `CDP_PORT`, default 9222) - to Google-search / play YouTube / read the page. The action is visible on the - Go-Live broadcast. The browser is already up on the VNC display `:1`. -- **false**: use Gemini (grounded with Google Search) for real-time info. No - screen share needed (voice + Gemini only). Auth sub-mode is `GEMINI_AUTH`: +`STREAM_BROWSER` is the broadcast **capability** master flag; the **live** +screen-share state then decides the search backend per turn: + +- **`STREAM_BROWSER=true`** (default): broadcasting is allowed. + - Joining a voice channel auto-starts the broadcast. + - While the broadcast is **live**: drive the on-screen Chrome (CDP at + `CDP_PORT`, default 9222) to Google-search / play YouTube / read the page — + visible on the Go-Live stream (VNC display `:1`). + - While the broadcast is **off**: use Gemini (grounded search), no screen needed. + - The user can toggle the broadcast on/off by voice ("방송 켜줘" / "방송 꺼줘"), + routed through an LLM tool (no hardcoded phrase matching). +- **`STREAM_BROWSER=false`**: broadcasting is disabled entirely. Any `/stream` + or "start broadcast" request is refused, and real-time search always uses + Gemini. Auth sub-mode is `GEMINI_AUTH`: - `oauth` (default): the Gemini CLI with a Google-account login (no API key). - `apikey`: the REST API keyed by `GEMINI_API_KEY`. +The brain receives the live broadcast state per request (`ToolContext.broadcasting`, +threaded from the bot via the bridge); `None` means "no signal" (evals, text +entry) and falls back to the master flag so behaviour is unchanged. + ## Components | Piece | Path | Status | @@ -21,8 +34,13 @@ info is fetched by a tool the reply engine calls. `STREAM_BROWSER` selects HOW: | Gemini auth mode | `GEMINI_AUTH` (oauth/apikey), `GEMINI_API_KEY`, `GEMINI_MODEL` (.env) + `config.py` | done | | `browseAndSearch` tool (true) | `src/jarvis/tools/builtin/browse_and_search.py` -> subprocess the Node core | TODO | | Gemini search (false) | `realtime_search.py` `gemini_cli_search()` (oauth, CLI) / `gemini_search()` (apikey, REST) | done | -| Registry (mode-gated) | `src/jarvis/tools/registry.py` `BUILTIN_TOOLS` | TODO | -| Specs + `docs/llm_contexts.md` | alongside each tool | TODO | +| Search routing by live broadcast | `web_search.py` uses `ToolContext.broadcasting` + master flag | done | +| Live broadcast state in tool ctx | `tools/base.py` `ToolContext.broadcasting` | done | +| Bridge passes broadcast state | bot `bridge.ts` request + `bridge/server.py` -> ToolContext | TODO | +| Auto-broadcast on voice join (true mode) | `bot/src/index.ts` `handleJoin()` starts streamer | TODO | +| Voice broadcast on/off toggle | brain tool -> reply directive -> bot `streamer.start/stop` | TODO | +| `/stream` + start refused (false mode) | `bot/src/index.ts` `handleStream()` already gated | done | +| Specs + `docs/llm_contexts.md` | alongside each tool | done | ## Design decisions diff --git a/src/jarvis/tools/base.py b/src/jarvis/tools/base.py index bb58f7f..3e1f584 100644 --- a/src/jarvis/tools/base.py +++ b/src/jarvis/tools/base.py @@ -22,6 +22,7 @@ class ToolContext: max_retries: int, user_print: Callable[[str], None], language: Optional[str] = None, + broadcasting: Optional[bool] = None, ): self.db = db self.cfg = cfg @@ -36,6 +37,13 @@ class ToolContext: # treat absence as "no signal" and fall back to their own default # rather than assuming English. self.language = language + # Live broadcast (screen-share / Go-Live) state for THIS turn, passed in + # by the bot per request. Controls real-time search routing when the + # master flag ``cfg.stream_browser`` is on: broadcasting -> on-screen + # Chrome search (visible on the stream), not broadcasting -> Gemini. + # ``None`` means "no signal" (evals, text entry, older bot): callers + # fall back to the master flag so behaviour is unchanged. + self.broadcasting = broadcasting class Tool(ABC): diff --git a/src/jarvis/tools/builtin/web_search.py b/src/jarvis/tools/builtin/web_search.py index 212e518..4202ff5 100644 --- a/src/jarvis/tools/builtin/web_search.py +++ b/src/jarvis/tools/builtin/web_search.py @@ -594,15 +594,27 @@ class WebSearchTool(Tool): context.user_print(f"🌐 Searching the web for '{search_query}'…") debug_log(f" 🌐 searching for '{search_query}'", "web") - # Real-time info routing by STREAM_BROWSER (docs/stream_browser_modes.md): - # true -> drive the on-screen Chrome (visible on the broadcast), - # false -> Gemini grounded search. Either falls through to the - # DDG/Brave/Wikipedia cascade below if it yields nothing (fail-open). + # Real-time info routing (docs/stream_browser_modes.md): + # master flag cfg.stream_browser (env STREAM_BROWSER) is the + # broadcast *capability*; context.broadcasting is the *live* + # screen-share state for this turn (set by the bot). + # - master off -> broadcast disabled, always Gemini + # - master on + live on -> on-screen Chrome (visible on stream) + # - master on + live off -> Gemini + # context.broadcasting is None outside the voice path (evals, text + # entry, older bot) -> fall back to the master flag so behaviour is + # unchanged. Either backend falls through to the DDG/Brave/Wikipedia + # cascade below if it yields nothing (fail-open). from .realtime_search import browser_search, gemini_search, gemini_cli_search - if getattr(cfg, "stream_browser", True): + master_browser = getattr(cfg, "stream_browser", True) + live = getattr(context, "broadcasting", None) + if live is None: + live = master_browser + use_browser = master_browser and live + if use_browser: routed = browser_search(search_query) if routed: - debug_log(" 🌐 routed via browser (STREAM_BROWSER=true)", "web") + debug_log(" 🌐 routed via browser (broadcast live)", "web") return ToolExecutionResult(success=True, reply_text=routed) elif getattr(cfg, "gemini_auth", "oauth") == "oauth": routed = gemini_cli_search(search_query) diff --git a/src/jarvis/tools/builtin/web_search.spec.md b/src/jarvis/tools/builtin/web_search.spec.md index 8957869..08e560e 100644 --- a/src/jarvis/tools/builtin/web_search.spec.md +++ b/src/jarvis/tools/builtin/web_search.spec.md @@ -7,14 +7,23 @@ memory. ### Real-time info routing (`STREAM_BROWSER`) -Before the DuckDuckGo cascade, `run()` routes by the env flag `STREAM_BROWSER` -(mirrored into `cfg.stream_browser`; see `docs/stream_browser_modes.md` and -`realtime_search.py`): +Before the DuckDuckGo cascade, `run()` routes by the broadcast capability flag +`cfg.stream_browser` (env `STREAM_BROWSER`) combined with the live broadcast +state `context.broadcasting` for the turn (see `docs/stream_browser_modes.md` +and `realtime_search.py`). `context.broadcasting` is `None` outside the voice +path (evals, text entry) and falls back to the master flag: -- **true** (default): `browser_search()` drives the on-screen Chrome (Node CDP - helper `bot/scripts/stream-test/browse-search.mjs`) to Google-search the - query, so the action is visible on the Go-Live broadcast. -- **false**: Gemini answers, with the sub-mode chosen by `cfg.gemini_auth` +| `cfg.stream_browser` | `context.broadcasting` | backend | +|---|---|---| +| True | True | on-screen Chrome (`browser_search()`) | +| True | False | Gemini | +| True | None | Chrome (no signal -> master flag) | +| False | any | Gemini (broadcast disabled, never Chrome) | + +- **on-screen Chrome**: `browser_search()` drives Chrome (Node CDP helper + `bot/scripts/stream-test/browse-search.mjs`) to Google-search the query, so + the action is visible on the Go-Live broadcast. +- **Gemini**: answers, with the sub-mode chosen by `cfg.gemini_auth` (env `GEMINI_AUTH`, default `oauth`): - `oauth` (default): `gemini_cli_search()` shells out to the Gemini CLI (`gemini -p -o json --skip-trust`, default approval mode) using the diff --git a/tests/test_web_search_broadcast_routing.py b/tests/test_web_search_broadcast_routing.py new file mode 100644 index 0000000..aa8ea80 --- /dev/null +++ b/tests/test_web_search_broadcast_routing.py @@ -0,0 +1,75 @@ +"""Behaviour tests for real-time search routing by live broadcast state. + +Routing contract (docs/stream_browser_modes.md): + - master flag cfg.stream_browser (env STREAM_BROWSER) = broadcast capability + - context.broadcasting = live screen-share state for the turn (set by the bot) + + master live(ctx) -> backend + ------- --------- ------- + True True on-screen Chrome (browser_search) + True False Gemini + True None Chrome (no signal -> fall back to master) + False True/None Gemini (broadcast disabled, never Chrome) +""" + +from types import SimpleNamespace + +import jarvis.tools.builtin.realtime_search as rs +from jarvis.tools.base import ToolContext +from jarvis.tools.builtin.web_search import WebSearchTool + + +def _ctx(stream_browser, broadcasting): + cfg = SimpleNamespace( + web_search_enabled=True, + stream_browser=stream_browser, + gemini_auth="oauth", + gemini_api_key="", + gemini_model="gemini-2.0-flash", + ) + return ToolContext( + db=None, + cfg=cfg, + system_prompt="", + original_prompt="", + redacted_text="", + max_retries=0, + user_print=lambda *a, **k: None, + language=None, + broadcasting=broadcasting, + ) + + +def _route(monkeypatch, stream_browser, broadcasting): + """Run webSearch and return which backend handled it ('browser'/'gemini').""" + monkeypatch.setattr(rs, "browser_search", lambda *a, **k: "BROWSER_RESULT") + monkeypatch.setattr(rs, "gemini_cli_search", lambda *a, **k: "GEMINI_RESULT") + monkeypatch.setattr(rs, "gemini_search", lambda *a, **k: "GEMINI_RESULT") + res = WebSearchTool().run({"search_query": "현재 서울 날씨"}, _ctx(stream_browser, broadcasting)) + if res.reply_text == "BROWSER_RESULT": + return "browser" + if res.reply_text == "GEMINI_RESULT": + return "gemini" + return "fallthrough" + + +def test_master_on_live_on_uses_browser(monkeypatch): + assert _route(monkeypatch, True, True) == "browser" + + +def test_master_on_live_off_uses_gemini(monkeypatch): + assert _route(monkeypatch, True, False) == "gemini" + + +def test_master_on_no_signal_defaults_to_browser(monkeypatch): + # None = no live signal (evals / text entry / older bot) -> behave as master + assert _route(monkeypatch, True, None) == "browser" + + +def test_master_off_never_uses_browser_even_if_live(monkeypatch): + # STREAM_BROWSER=false disables broadcast entirely: always Gemini. + assert _route(monkeypatch, False, True) == "gemini" + + +def test_master_off_no_signal_uses_gemini(monkeypatch): + assert _route(monkeypatch, False, None) == "gemini"