Files
javis_bot/tests/test_voice_worker_collect.py
javis-bot 69f1bf4fa9 fix(app): collection fetches addresses with Korean paths and words
The request line is ASCII: other characters and spaces in the path and query go as UTF-8 %HH,
while existing %HH and separators stay.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 21:49:01 +09:00

386 lines
15 KiB
Python

"""Background collection (voice_worker.spec.md, "Background collection").
The collector runs against a fake fetcher (the real one refuses loopback addresses, which is
checked on its own). Behaviours: each kind keeps only what is new, a target runs again only after
its interval (also across a restart), a failure counts as a run and logs no URL, old day files go,
and nothing collected reaches the log.
"""
from __future__ import annotations
import http.server
import ipaddress
import json
import socket
import threading
import time
from datetime import datetime, timedelta
from pathlib import Path
import pytest
import voice_worker.collect as collect
from voice_worker.collect import Collector, load_config, safe_get
RSS = """<?xml version="1.0"?><rss version="2.0"><channel><title>feed</title>
<item><title>First</title><link>https://example.com/1</link><guid>id-1</guid>
<pubDate>Fri, 13 Nov 2026 10:00:00 +0900</pubDate><description>&lt;p&gt;one &lt;b&gt;bold&lt;/b&gt;&lt;/p&gt;</description></item>
{more}
</channel></rss>"""
SECOND = """<item><title>Second</title><link>https://example.com/2</link><guid>id-2</guid></item>"""
ATOM = """<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"><title>a</title>
<entry><title>Atom one</title><id>tag:a,1</id><link href="https://example.org/a1"/>
<updated>2026-11-13T01:00:00Z</updated><summary>sum</summary></entry></feed>"""
DDG = """<html><body><a href="https://duckduckgo.com/settings">Settings</a>
<a href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fnews.example%2Fa&amp;rut=x">A first result title here</a>
<a href="https://news.example/b">Another result title here</a>{more}</body></html>"""
PAGE = "<html><head><title>Status</title><script>var x=1;</script></head><body><p>Version {v} is out</p></body></html>"
class Fake:
"""url -> bytes (or an exception to raise); records what was asked."""
def __init__(self, pages):
self.pages = pages
self.asked = []
def __call__(self, url):
self.asked.append(url)
v = self.pages[url]
if isinstance(v, Exception):
raise v
return v.encode() if isinstance(v, str) else v
def config(tmp_path, targets, keep_days=30):
p = tmp_path / "collect.json"
p.write_text(json.dumps({"dir": str(tmp_path / "collect"), "keepDays": keep_days, "targets": targets}))
return load_config(p)
def items(tmp_path, name):
out = []
for f in sorted((tmp_path / "collect" / name).glob("*.jsonl")):
out += [json.loads(line) for line in f.read_text(encoding="utf-8").splitlines()]
return out
def lines(capfd):
return capfd.readouterr().err
def test_rss_keeps_each_new_item_once(tmp_path, capfd):
fake = Fake({"https://feed.example/rss": RSS.format(more="")})
cfg = config(tmp_path, [{"name": "뉴스", "kind": "rss", "value": "https://feed.example/rss", "everyMinutes": 15}])
c = Collector(cfg, fetch=fake)
c.run_due(now=1000.0)
got = items(tmp_path, "뉴스")
assert [i["title"] for i in got] == ["First"]
assert got[0]["url"] == "https://example.com/1"
assert got[0]["text"] == "one bold"
assert got[0]["published"] == "Fri, 13 Nov 2026 10:00:00 +0900"
fake.pages["https://feed.example/rss"] = RSS.format(more=SECOND)
c.run_due(now=1000.0 + 15 * 60)
assert [i["title"] for i in items(tmp_path, "뉴스")] == ["First", "Second"]
out = lines(capfd)
assert "collect 뉴스: 1 new" in out
assert "feed.example" not in out and "First" not in out
def test_atom_feeds_are_read_too(tmp_path):
fake = Fake({"https://a.example/atom": ATOM})
c = Collector(config(tmp_path, [{"name": "a", "kind": "rss", "value": "https://a.example/atom", "everyMinutes": 60}]),
fetch=fake)
c.run_due(now=0.0)
[item] = items(tmp_path, "a")
assert (item["title"], item["url"], item["text"]) == ("Atom one", "https://example.org/a1", "sum")
def test_search_keeps_new_results_for_the_words(tmp_path):
fake = Fake({})
url = "https://lite.duckduckgo.com/lite/?q=rust+release"
fake.pages[url] = DDG.format(more="")
c = Collector(config(tmp_path, [{"name": "s", "kind": "search", "value": "rust release", "everyMinutes": 60}]),
fetch=fake)
c.run_due(now=0.0)
assert [(i["title"], i["url"]) for i in items(tmp_path, "s")] == [
("A first result title here", "https://news.example/a"), ("Another result title here", "https://news.example/b")]
fake.pages[url] = DDG.format(more='<a href="https://news.example/c">A third result title</a>')
c.run_due(now=3600.0)
assert [i["url"] for i in items(tmp_path, "s")][-1] == "https://news.example/c"
assert len(items(tmp_path, "s")) == 3
def test_a_search_challenge_page_is_a_failure_not_zero_results(tmp_path, capfd):
url = "https://lite.duckduckgo.com/lite/?q=words"
fake = Fake({url: '<html><div class="anomaly-modal"></div><script src="//duckduckgo.com/anomaly.js"></script></html>'})
c = Collector(config(tmp_path, [{"name": "s", "kind": "search", "value": "words", "everyMinutes": 60}]), fetch=fake)
c.run_due(now=0.0)
assert "Warning: collect s failed: SearchBlocked" in lines(capfd)
assert not list((tmp_path / "collect").glob("s/*.jsonl"))
def test_a_site_is_kept_only_when_it_changed(tmp_path):
fake = Fake({"https://s.example/": PAGE.format(v=1)})
c = Collector(config(tmp_path, [{"name": "site", "kind": "site", "value": "https://s.example/", "everyMinutes": 15}]),
fetch=fake)
c.run_due(now=0.0)
c.run_due(now=900.0)
got = items(tmp_path, "site")
assert len(got) == 1 and got[0]["title"] == "Status" and got[0]["text"] == "Version 1 is out"
fake.pages["https://s.example/"] = PAGE.format(v=2)
c.run_due(now=1800.0)
assert [i["text"] for i in items(tmp_path, "site")] == ["Version 1 is out", "Version 2 is out"]
def test_a_site_whose_title_alone_changed_is_kept_again(tmp_path):
page = "<html><head><title>{t}</title></head><body><p>Same body text</p></body></html>"
fake = Fake({"https://s.example/": page.format(t="Old")})
c = Collector(config(tmp_path, [{"name": "site", "kind": "site", "value": "https://s.example/", "everyMinutes": 15}]),
fetch=fake)
c.run_due(now=0.0)
fake.pages["https://s.example/"] = page.format(t="New")
c.run_due(now=900.0)
assert [i["title"] for i in items(tmp_path, "site")] == ["Old", "New"]
def test_a_target_runs_again_only_after_its_interval_also_after_a_restart(tmp_path):
fake = Fake({"https://s.example/": PAGE.format(v=1)})
targets = [{"name": "site", "kind": "site", "value": "https://s.example/", "everyMinutes": 60}]
Collector(config(tmp_path, targets), fetch=fake).run_due(now=10_000.0)
again = Collector(config(tmp_path, targets), fetch=fake)
again.run_due(now=10_000.0 + 59 * 60)
assert len(fake.asked) == 1
again.run_due(now=10_000.0 + 60 * 60)
assert len(fake.asked) == 2
def test_a_failure_counts_as_a_run_and_logs_no_address(tmp_path, capfd):
fake = Fake({"https://secret-site.example/": ConnectionError("down at https://secret-site.example/")})
c = Collector(config(tmp_path, [{"name": "x", "kind": "site", "value": "https://secret-site.example/", "everyMinutes": 15}]),
fetch=fake)
c.run_due(now=0.0)
c.run_due(now=60.0)
assert len(fake.asked) == 1
out = lines(capfd)
assert "Warning: collect x failed: ConnectionError" in out
assert "secret-site" not in out
def test_old_day_files_go_in_every_folder(tmp_path):
cfg = config(tmp_path, [], keep_days=2)
today = datetime.now().date()
for name in ("listed", "removed"):
d = tmp_path / "collect" / name
d.mkdir(parents=True)
for age in (0, 2, 3):
(d / f"{today - timedelta(days=age)}.jsonl").write_text("{}\n")
(d / "state.json").write_text("{}")
Collector(cfg, fetch=Fake({})).run_due(now=time.time())
for name in ("listed", "removed"):
left = sorted(p.name for p in (tmp_path / "collect" / name).iterdir())
assert left == sorted([f"{today - timedelta(days=2)}.jsonl", f"{today}.jsonl", "state.json"])
def test_the_real_fetcher_refuses_addresses_that_are_not_public():
for url in ("http://127.0.0.1:1/", "http://192.168.0.1/", "file:///etc/passwd", "http://localhost/"):
with pytest.raises(ValueError):
safe_get(url)
def test_the_thread_collects_on_its_own_and_stops(tmp_path):
fake = Fake({"https://s.example/": PAGE.format(v=1)})
c = Collector(config(tmp_path, [{"name": "site", "kind": "site", "value": "https://s.example/", "everyMinutes": 15}]),
fetch=fake, check_every=0.05)
c.start()
deadline = time.monotonic() + 5
while not list((tmp_path / "collect").glob("site/*.jsonl")):
assert time.monotonic() < deadline
time.sleep(0.02)
c.stop()
assert not c.running()
# ---- the real fetcher, against local servers -----------------------------------------------------
def serve(handler):
"""A local HTTP server on 127.0.0.1; returns (port, hits list)."""
hits = []
class H(http.server.BaseHTTPRequestHandler):
def do_GET(self):
hits.append(self.path)
handler(self)
def log_message(self, *a):
pass
srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), H)
srv.daemon_threads = True
threading.Thread(target=srv.serve_forever, daemon=True).start()
return srv.server_address[1], hits
def answer(body, status=200, headers=()):
def h(req):
req.send_response(status)
for k, v in headers:
req.send_header(k, v)
req.send_header("Content-Length", str(len(body)))
req.end_headers()
req.wfile.write(body)
return h
@pytest.fixture
def loopback_is_public(monkeypatch):
"""Lets the fetcher reach the local test servers; every other private address stays refused."""
real = collect._is_public_ip
monkeypatch.setattr(collect, "_is_public_ip", lambda ip: ip == ipaddress.ip_address("127.0.0.1") or real(ip))
def test_the_fetcher_follows_redirects_and_checks_every_hop(loopback_is_public):
def h(req):
if req.path == "/start":
answer(b"", 302, [("Location", "/end?x=1")])(req)
elif req.path == "/private":
answer(b"", 302, [("Location", "http://10.0.0.1/")])(req)
else:
answer(b"the body")(req)
port, hits = serve(h)
assert safe_get(f"http://127.0.0.1:{port}/start") == b"the body"
assert hits == ["/start", "/end?x=1"]
with pytest.raises(ValueError):
safe_get(f"http://127.0.0.1:{port}/private")
def test_the_fetcher_connects_to_the_address_it_checked(monkeypatch):
"""A name that resolves to a public address when checked and to a private one afterwards
(DNS rebinding) never reaches the private one."""
port, hits = serve(answer(b"internal"))
calls = []
def resolve(host, *a, **k):
if host == "rebind.example":
calls.append(host)
ip = "1.1.1.1" if len(calls) == 1 else "127.0.0.1"
else: # an address resolves to itself
ip = host
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, port))]
monkeypatch.setattr(socket, "getaddrinfo", resolve)
monkeypatch.setattr(collect, "CONNECT_S", 0.3)
with pytest.raises(Exception):
safe_get(f"http://rebind.example:{port}/")
assert hits == []
def test_the_fetcher_tries_the_next_checked_address_when_one_does_not_connect(loopback_is_public, monkeypatch):
port, hits = serve(answer(b"from the second address"))
real = socket.getaddrinfo
monkeypatch.setattr(socket, "getaddrinfo", lambda host, *a, **k: [
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("1.1.1.1", port)),
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", port))] if host == "two.example" else real(host, *a, **k))
monkeypatch.setattr(collect, "CONNECT_S", 0.3)
assert safe_get(f"http://two.example:{port}/") == b"from the second address"
assert hits == ["/"]
def test_the_fetcher_stops_a_server_that_sends_slowly(loopback_is_public, monkeypatch):
def drip(req):
req.send_response(200)
req.send_header("Content-Length", "100000")
req.end_headers()
try:
for _ in range(100):
req.wfile.write(b"x")
req.wfile.flush()
time.sleep(0.1)
except OSError:
pass
port, _ = serve(drip)
monkeypatch.setattr(collect, "TOTAL_S", 1.0)
monkeypatch.setattr(collect, "READ_S", 0.5)
t0 = time.monotonic()
with pytest.raises(Exception):
safe_get(f"http://127.0.0.1:{port}/")
assert time.monotonic() - t0 < 1.5
def test_the_fetcher_never_logs_the_address(monkeypatch, capfd):
import jarvis.debug
monkeypatch.setattr(jarvis.debug, "_is_debug_enabled", lambda: True)
monkeypatch.setattr(socket, "getaddrinfo",
lambda host, *a, **k: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 80))])
with pytest.raises(ValueError):
safe_get("http://secret-host.example/private/path?token=abc123")
out = capfd.readouterr()
assert "secret-host" not in out.err + out.out and "abc123" not in out.err + out.out
def test_addresses_with_korean_paths_and_words_are_fetched(loopback_is_public):
port, hits = serve(answer(b"ok"))
assert safe_get(f"http://127.0.0.1:{port}/wiki/대한민국?q=한글 검색&x=%2F#top") == b"ok"
from urllib.parse import quote
assert hits == [f"/wiki/{quote('대한민국')}?q={quote('한글')}%20{quote('검색')}&x=%2F"]
def test_a_response_cut_short_is_a_failure_not_a_page(loopback_is_public):
def short(req):
req.send_response(200)
req.send_header("Content-Length", "1000")
req.end_headers()
req.wfile.write(b"only part")
req.close_connection = True
port, _ = serve(short)
with pytest.raises(Exception):
safe_get(f"http://127.0.0.1:{port}/")
def test_the_fetcher_stops_a_server_that_sends_its_headers_slowly(loopback_is_public, monkeypatch):
"""Each byte comes before the per-read limit, so only the overall limit can end it."""
srv = socket.socket()
srv.bind(("127.0.0.1", 0))
srv.listen()
def drip():
conn, _ = srv.accept()
try:
for b in b"HTTP/1.1 200 OK\r\nX-Slow: " + b"x" * 1000:
conn.sendall(bytes([b]))
time.sleep(0.2)
except OSError:
pass
finally:
conn.close()
threading.Thread(target=drip, daemon=True).start()
monkeypatch.setattr(collect, "TOTAL_S", 1.0)
monkeypatch.setattr(collect, "READ_S", 0.5)
t0 = time.monotonic()
with pytest.raises(Exception):
safe_get(f"http://127.0.0.1:{srv.getsockname()[1]}/")
took = time.monotonic() - t0
print(f"slow headers: failed after {took:.2f}s")
assert took < 1.5
srv.close()
def test_the_fetcher_gives_up_on_a_slow_name_lookup(monkeypatch):
def slow(host, *a, **k):
time.sleep(3)
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("1.1.1.1", 80))]
monkeypatch.setattr(socket, "getaddrinfo", slow)
monkeypatch.setattr(collect, "TOTAL_S", 1.0)
monkeypatch.setattr(collect, "READ_S", 0.5)
t0 = time.monotonic()
with pytest.raises(TimeoutError):
safe_get("http://slow-dns.example/")
took = time.monotonic() - t0
print(f"slow lookup: failed after {took:.2f}s")
assert took < 1.5