feat: play the Nth YouTube result in browseAndPlay via an index arg
Some checks failed
Release / semantic-release (push) Successful in 22s
tests / Unit tests (Linux, Python 3.11) (push) Failing after 5m16s
Release / build-linux (push) Failing after 7m10s
Release / build-windows (push) Has been cancelled
Release / build-macos (arm64, macos-latest) (push) Has been cancelled
Release / build-macos (x64, macos-15-intel) (push) Has been cancelled
Release / release-main (push) Has been cancelled
Release / release-develop (push) Has been cancelled
Some checks failed
Release / semantic-release (push) Successful in 22s
tests / Unit tests (Linux, Python 3.11) (push) Failing after 5m16s
Release / build-linux (push) Failing after 7m10s
Release / build-windows (push) Has been cancelled
Release / build-macos (arm64, macos-latest) (push) Has been cancelled
Release / build-macos (x64, macos-15-intel) (push) Has been cancelled
Release / release-main (push) Has been cancelled
Release / release-develop (push) Has been cancelled
agents/llm.md promises "play the Nth video from the top", but browseAndPlay only ever clicked the first result. Add an optional 1-based index argument (default 1, backward-compatible) threaded to the Node helper, which now clicks the Nth a#video-title and clamps to the number of results returned so asking beyond the list plays the last available video instead of failing. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -2,10 +2,11 @@
|
|||||||
// 9222) so the action is visible on the Go-Live broadcast, and prints a JSON
|
// 9222) so the action is visible on the Go-Live broadcast, and prints a JSON
|
||||||
// result on stdout for the Python `browseAndSearch` tool to wrap.
|
// result on stdout for the Python `browseAndSearch` tool to wrap.
|
||||||
//
|
//
|
||||||
// node browse-search.mjs "<query>" [search|youtube]
|
// node browse-search.mjs "<query>" [search|youtube] [index]
|
||||||
//
|
//
|
||||||
// - search : Google-search the query, return the top organic results.
|
// - search : Google-search the query, return the top organic results.
|
||||||
// - youtube : search YouTube and play the first result.
|
// - youtube : search YouTube and play a result. `index` is the 1-based position
|
||||||
|
// from the top of the result list (default 1 = first result).
|
||||||
//
|
//
|
||||||
// Backend selection for `search`:
|
// Backend selection for `search`:
|
||||||
// 1. The broadcast Chrome over CDP (visible on the Go-Live stream).
|
// 1. The broadcast Chrome over CDP (visible on the Go-Live stream).
|
||||||
@@ -29,6 +30,9 @@ const UA =
|
|||||||
'(KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36';
|
'(KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36';
|
||||||
const query = process.argv[2] || '';
|
const query = process.argv[2] || '';
|
||||||
const mode = (process.argv[3] || 'search').toLowerCase();
|
const mode = (process.argv[3] || 'search').toLowerCase();
|
||||||
|
// 1-based position of the YouTube result to play, counted from the top of the
|
||||||
|
// list. Defaults to 1 (first result). Anything <1 or non-numeric falls back to 1.
|
||||||
|
const playIndex = Math.max(1, parseInt(process.argv[4], 10) || 1);
|
||||||
const out = (o) => { process.stdout.write(JSON.stringify(o)); };
|
const out = (o) => { process.stdout.write(JSON.stringify(o)); };
|
||||||
|
|
||||||
if (!query) { out({ ok: false, error: 'no query' }); process.exit(1); }
|
if (!query) { out({ ok: false, error: 'no query' }); process.exit(1); }
|
||||||
@@ -105,15 +109,21 @@ try {
|
|||||||
await page.bringToFront().catch(() => {});
|
await page.bringToFront().catch(() => {});
|
||||||
|
|
||||||
if (mode === 'youtube') {
|
if (mode === 'youtube') {
|
||||||
// Type into YouTube's search box like a person, then play the first result.
|
// Type into YouTube's search box like a person, then play the requested
|
||||||
|
// result (the Nth from the top of the list; default the first).
|
||||||
await typeSearch('https://www.youtube.com/?hl=ko', 'input#search, input[name="search_query"]', query);
|
await typeSearch('https://www.youtube.com/?hl=ko', 'input#search, input[name="search_query"]', query);
|
||||||
await page.waitForSelector('ytd-video-renderer a#video-title, a#video-title', { timeout: 20000 });
|
await page.waitForSelector('ytd-video-renderer a#video-title, a#video-title', { timeout: 20000 });
|
||||||
const first = page.locator('ytd-video-renderer a#video-title, a#video-title').first();
|
const results = page.locator('ytd-video-renderer a#video-title, a#video-title');
|
||||||
const title = (await first.getAttribute('title').catch(() => '')) || (await first.innerText().catch(() => ''));
|
// Clamp to what's actually on the page so "play the 5th" still plays the
|
||||||
await first.click();
|
// last available result rather than failing when fewer were returned.
|
||||||
|
const available = await results.count();
|
||||||
|
const targetIdx = Math.min(playIndex, Math.max(available, 1)) - 1;
|
||||||
|
const target = results.nth(targetIdx);
|
||||||
|
const title = (await target.getAttribute('title').catch(() => '')) || (await target.innerText().catch(() => ''));
|
||||||
|
await target.click();
|
||||||
await page.waitForSelector('#movie_player', { timeout: 20000 });
|
await page.waitForSelector('#movie_player', { timeout: 20000 });
|
||||||
await page.evaluate(() => { const v = document.querySelector('video'); if (v && v.paused) v.play(); });
|
await page.evaluate(() => { const v = document.querySelector('video'); if (v && v.paused) v.play(); });
|
||||||
out({ ok: true, mode, title: (title || '').trim(), url: page.url() });
|
out({ ok: true, mode, index: targetIdx + 1, title: (title || '').trim(), url: page.url() });
|
||||||
} else {
|
} else {
|
||||||
// Type into Google's search box like a person, then read the results.
|
// Type into Google's search box like a person, then read the results.
|
||||||
await typeSearch('https://www.google.com/?hl=ko', 'textarea[name="q"], input[name="q"]', query);
|
await typeSearch('https://www.google.com/?hl=ko', 'textarea[name="q"], input[name="q"]', query);
|
||||||
|
|||||||
@@ -30,8 +30,10 @@ class BrowseAndPlayTool(Tool):
|
|||||||
def description(self) -> str:
|
def description(self) -> str:
|
||||||
return (
|
return (
|
||||||
"Play a song / music video / clip on the shared screen by searching YouTube "
|
"Play a song / music video / clip on the shared screen by searching YouTube "
|
||||||
"and playing the first result. Use when the user asks you to play or watch "
|
"and playing a result. Use when the user asks you to play or watch "
|
||||||
"something. Only available in screen-share mode."
|
"something. Plays the first result by default; pass 'index' to play the "
|
||||||
|
"Nth result from the top of the search list (e.g. 'play the 3rd video' -> "
|
||||||
|
"index=3). Only available in screen-share mode."
|
||||||
)
|
)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
@@ -42,7 +44,16 @@ class BrowseAndPlayTool(Tool):
|
|||||||
"query": {
|
"query": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"description": "What to play, e.g. 'IU Good Day' or 'lofi hip hop'.",
|
"description": "What to play, e.g. 'IU Good Day' or 'lofi hip hop'.",
|
||||||
}
|
},
|
||||||
|
"index": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": (
|
||||||
|
"1-based position of the video to play in the search results, "
|
||||||
|
"counted from the top of the list. Defaults to 1 (first result). "
|
||||||
|
"Use for 'play the Nth video' / 'play the second one'."
|
||||||
|
),
|
||||||
|
"minimum": 1,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
"required": ["query"],
|
"required": ["query"],
|
||||||
}
|
}
|
||||||
@@ -55,18 +66,25 @@ class BrowseAndPlayTool(Tool):
|
|||||||
reply_text="화면 공유 모드(STREAM_BROWSER=true)에서만 영상을 재생할 수 있습니다.",
|
reply_text="화면 공유 모드(STREAM_BROWSER=true)에서만 영상을 재생할 수 있습니다.",
|
||||||
)
|
)
|
||||||
query = ""
|
query = ""
|
||||||
|
index = 1
|
||||||
if args and isinstance(args, dict):
|
if args and isinstance(args, dict):
|
||||||
query = str(args.get("query", "")).strip()
|
query = str(args.get("query", "")).strip()
|
||||||
|
try:
|
||||||
|
index = int(args.get("index", 1) or 1)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
index = 1
|
||||||
|
if index < 1:
|
||||||
|
index = 1
|
||||||
if not query:
|
if not query:
|
||||||
return ToolExecutionResult(success=False, reply_text="재생할 내용을 알려주세요.")
|
return ToolExecutionResult(success=False, reply_text="재생할 내용을 알려주세요.")
|
||||||
if not _NODE_SCRIPT.exists():
|
if not _NODE_SCRIPT.exists():
|
||||||
return ToolExecutionResult(success=False, reply_text="브라우저 재생 도구를 찾을 수 없습니다.")
|
return ToolExecutionResult(success=False, reply_text="브라우저 재생 도구를 찾을 수 없습니다.")
|
||||||
|
|
||||||
context.user_print(f"▶️ 화면에서 '{query}' 재생 중…")
|
context.user_print(f"▶️ 화면에서 '{query}' 재생 중… (#{index})")
|
||||||
debug_log(f" ▶️ browseAndPlay '{query}'", "tools")
|
debug_log(f" ▶️ browseAndPlay '{query}' index={index}", "tools")
|
||||||
try:
|
try:
|
||||||
proc = subprocess.run(
|
proc = subprocess.run(
|
||||||
["node", str(_NODE_SCRIPT), query, "youtube"],
|
["node", str(_NODE_SCRIPT), query, "youtube", str(index)],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
timeout=40,
|
timeout=40,
|
||||||
|
|||||||
@@ -6,16 +6,24 @@ video, or clip.
|
|||||||
|
|
||||||
### Behaviour
|
### Behaviour
|
||||||
|
|
||||||
- Public schema is a single required `query` string (what to play).
|
- Public schema is a required `query` string (what to play) plus an optional
|
||||||
|
`index` integer (1-based position in the search results, counted from the top
|
||||||
|
of the list). `index` defaults to `1` (first result), so existing callers and
|
||||||
|
"play X" requests are unchanged; "play the 3rd video" / "play the second one"
|
||||||
|
map to `index=3` / `index=2`.
|
||||||
- **Mode-gated**: only acts when `STREAM_BROWSER` is true (`cfg.stream_browser`).
|
- **Mode-gated**: only acts when `STREAM_BROWSER` is true (`cfg.stream_browser`).
|
||||||
In voice-only mode (false) there is no screen to show, so it returns a short
|
In voice-only mode (false) there is no screen to show, so it returns a short
|
||||||
message and does nothing.
|
message and does nothing.
|
||||||
- Drives the on-screen Chrome by subprocessing the Node CDP helper
|
- Drives the on-screen Chrome by subprocessing the Node CDP helper
|
||||||
`bot/scripts/stream-test/browse-search.mjs <query> youtube`, which searches
|
`bot/scripts/stream-test/browse-search.mjs <query> youtube <index>`, which
|
||||||
YouTube and plays the first result on display `:1`. The broadcast captures
|
searches YouTube and plays the chosen result on display `:1`. The broadcast
|
||||||
that display, so the playback is what viewers see.
|
captures that display, so the playback is what viewers see.
|
||||||
- Returns `success` with the played video's title, or a failure message if the
|
- The helper clicks the `index`-th `a#video-title` in the results list. The
|
||||||
helper/Chrome is unavailable. It does NOT make an LLM call.
|
index is clamped to the number of results actually returned, so asking for a
|
||||||
|
position beyond the list plays the last available result rather than failing.
|
||||||
|
- Returns `success` with the played video's title (and the resolved `index`), or
|
||||||
|
a failure message if the helper/Chrome is unavailable. It does NOT make an LLM
|
||||||
|
call.
|
||||||
|
|
||||||
### Principles
|
### Principles
|
||||||
|
|
||||||
|
|||||||
79
tests/test_browse_and_play_index.py
Normal file
79
tests/test_browse_and_play_index.py
Normal file
@@ -0,0 +1,79 @@
|
|||||||
|
"""Tests for browseAndPlay's ``index`` argument (play the Nth search result).
|
||||||
|
|
||||||
|
Behaviour verified:
|
||||||
|
- default plays the first result (index 1) and stays backward-compatible,
|
||||||
|
- an explicit index is forwarded to the Node helper as the 4th argv,
|
||||||
|
- bad / sub-1 index values clamp to 1,
|
||||||
|
- the index is advertised in the tool schema.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from src.jarvis.tools.builtin.browse_and_play import BrowseAndPlayTool, _NODE_SCRIPT
|
||||||
|
|
||||||
|
|
||||||
|
def _ctx():
|
||||||
|
cfg = Mock()
|
||||||
|
cfg.stream_browser = True
|
||||||
|
return Mock(cfg=cfg, user_print=Mock())
|
||||||
|
|
||||||
|
|
||||||
|
def _run(args):
|
||||||
|
tool = BrowseAndPlayTool()
|
||||||
|
with patch("src.jarvis.tools.builtin.browse_and_play.subprocess.run") as mock_run:
|
||||||
|
mock_run.return_value = Mock(
|
||||||
|
stdout=json.dumps({"ok": True, "title": "Some Video"}),
|
||||||
|
stderr="",
|
||||||
|
)
|
||||||
|
result = tool.run(args, _ctx())
|
||||||
|
return mock_run, result
|
||||||
|
|
||||||
|
|
||||||
|
def _argv(mock_run):
|
||||||
|
return list(mock_run.call_args[0][0])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_schema_exposes_index():
|
||||||
|
schema = BrowseAndPlayTool().inputSchema
|
||||||
|
assert "index" in schema["properties"]
|
||||||
|
assert schema["properties"]["index"]["type"] == "integer"
|
||||||
|
assert "query" in schema["required"]
|
||||||
|
assert "index" not in schema["required"] # optional
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_default_index_is_first():
|
||||||
|
mock_run, result = _run({"query": "IU Good Day"})
|
||||||
|
argv = _argv(mock_run)
|
||||||
|
assert argv[:4] == ["node", str(_NODE_SCRIPT), "IU Good Day", "youtube"]
|
||||||
|
assert argv[4] == "1"
|
||||||
|
assert result.success is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_explicit_index_forwarded():
|
||||||
|
mock_run, _ = _run({"query": "lofi", "index": 3})
|
||||||
|
assert _argv(mock_run)[4] == "3"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
@pytest.mark.parametrize("bad", [0, -2, "nope", None])
|
||||||
|
def test_bad_index_clamps_to_one(bad):
|
||||||
|
mock_run, _ = _run({"query": "lofi", "index": bad})
|
||||||
|
assert _argv(mock_run)[4] == "1"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_voice_only_mode_does_not_play():
|
||||||
|
tool = BrowseAndPlayTool()
|
||||||
|
cfg = Mock()
|
||||||
|
cfg.stream_browser = False
|
||||||
|
ctx = Mock(cfg=cfg, user_print=Mock())
|
||||||
|
with patch("src.jarvis.tools.builtin.browse_and_play.subprocess.run") as mock_run:
|
||||||
|
result = tool.run({"query": "x", "index": 2}, ctx)
|
||||||
|
assert result.success is False
|
||||||
|
mock_run.assert_not_called()
|
||||||
Reference in New Issue
Block a user