The request line is ASCII: other characters and spaces in the path and query go as UTF-8 %HH, while existing %HH and separators stay. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
386 lines
15 KiB
Python
386 lines
15 KiB
Python
"""Background collection (voice_worker.spec.md, "Background collection").
|
|
|
|
The collector runs against a fake fetcher (the real one refuses loopback addresses, which is
|
|
checked on its own). Behaviours: each kind keeps only what is new, a target runs again only after
|
|
its interval (also across a restart), a failure counts as a run and logs no URL, old day files go,
|
|
and nothing collected reaches the log.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import http.server
|
|
import ipaddress
|
|
import json
|
|
import socket
|
|
import threading
|
|
import time
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
import voice_worker.collect as collect
|
|
from voice_worker.collect import Collector, load_config, safe_get
|
|
|
|
RSS = """<?xml version="1.0"?><rss version="2.0"><channel><title>feed</title>
|
|
<item><title>First</title><link>https://example.com/1</link><guid>id-1</guid>
|
|
<pubDate>Fri, 13 Nov 2026 10:00:00 +0900</pubDate><description><p>one <b>bold</b></p></description></item>
|
|
{more}
|
|
</channel></rss>"""
|
|
SECOND = """<item><title>Second</title><link>https://example.com/2</link><guid>id-2</guid></item>"""
|
|
|
|
ATOM = """<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"><title>a</title>
|
|
<entry><title>Atom one</title><id>tag:a,1</id><link href="https://example.org/a1"/>
|
|
<updated>2026-11-13T01:00:00Z</updated><summary>sum</summary></entry></feed>"""
|
|
|
|
DDG = """<html><body><a href="https://duckduckgo.com/settings">Settings</a>
|
|
<a href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fnews.example%2Fa&rut=x">A first result title here</a>
|
|
<a href="https://news.example/b">Another result title here</a>{more}</body></html>"""
|
|
|
|
PAGE = "<html><head><title>Status</title><script>var x=1;</script></head><body><p>Version {v} is out</p></body></html>"
|
|
|
|
|
|
class Fake:
|
|
"""url -> bytes (or an exception to raise); records what was asked."""
|
|
|
|
def __init__(self, pages):
|
|
self.pages = pages
|
|
self.asked = []
|
|
|
|
def __call__(self, url):
|
|
self.asked.append(url)
|
|
v = self.pages[url]
|
|
if isinstance(v, Exception):
|
|
raise v
|
|
return v.encode() if isinstance(v, str) else v
|
|
|
|
|
|
def config(tmp_path, targets, keep_days=30):
|
|
p = tmp_path / "collect.json"
|
|
p.write_text(json.dumps({"dir": str(tmp_path / "collect"), "keepDays": keep_days, "targets": targets}))
|
|
return load_config(p)
|
|
|
|
|
|
def items(tmp_path, name):
|
|
out = []
|
|
for f in sorted((tmp_path / "collect" / name).glob("*.jsonl")):
|
|
out += [json.loads(line) for line in f.read_text(encoding="utf-8").splitlines()]
|
|
return out
|
|
|
|
|
|
def lines(capfd):
|
|
return capfd.readouterr().err
|
|
|
|
|
|
def test_rss_keeps_each_new_item_once(tmp_path, capfd):
|
|
fake = Fake({"https://feed.example/rss": RSS.format(more="")})
|
|
cfg = config(tmp_path, [{"name": "뉴스", "kind": "rss", "value": "https://feed.example/rss", "everyMinutes": 15}])
|
|
c = Collector(cfg, fetch=fake)
|
|
c.run_due(now=1000.0)
|
|
got = items(tmp_path, "뉴스")
|
|
assert [i["title"] for i in got] == ["First"]
|
|
assert got[0]["url"] == "https://example.com/1"
|
|
assert got[0]["text"] == "one bold"
|
|
assert got[0]["published"] == "Fri, 13 Nov 2026 10:00:00 +0900"
|
|
fake.pages["https://feed.example/rss"] = RSS.format(more=SECOND)
|
|
c.run_due(now=1000.0 + 15 * 60)
|
|
assert [i["title"] for i in items(tmp_path, "뉴스")] == ["First", "Second"]
|
|
out = lines(capfd)
|
|
assert "collect 뉴스: 1 new" in out
|
|
assert "feed.example" not in out and "First" not in out
|
|
|
|
|
|
def test_atom_feeds_are_read_too(tmp_path):
|
|
fake = Fake({"https://a.example/atom": ATOM})
|
|
c = Collector(config(tmp_path, [{"name": "a", "kind": "rss", "value": "https://a.example/atom", "everyMinutes": 60}]),
|
|
fetch=fake)
|
|
c.run_due(now=0.0)
|
|
[item] = items(tmp_path, "a")
|
|
assert (item["title"], item["url"], item["text"]) == ("Atom one", "https://example.org/a1", "sum")
|
|
|
|
|
|
def test_search_keeps_new_results_for_the_words(tmp_path):
|
|
fake = Fake({})
|
|
url = "https://lite.duckduckgo.com/lite/?q=rust+release"
|
|
fake.pages[url] = DDG.format(more="")
|
|
c = Collector(config(tmp_path, [{"name": "s", "kind": "search", "value": "rust release", "everyMinutes": 60}]),
|
|
fetch=fake)
|
|
c.run_due(now=0.0)
|
|
assert [(i["title"], i["url"]) for i in items(tmp_path, "s")] == [
|
|
("A first result title here", "https://news.example/a"), ("Another result title here", "https://news.example/b")]
|
|
fake.pages[url] = DDG.format(more='<a href="https://news.example/c">A third result title</a>')
|
|
c.run_due(now=3600.0)
|
|
assert [i["url"] for i in items(tmp_path, "s")][-1] == "https://news.example/c"
|
|
assert len(items(tmp_path, "s")) == 3
|
|
|
|
|
|
def test_a_search_challenge_page_is_a_failure_not_zero_results(tmp_path, capfd):
|
|
url = "https://lite.duckduckgo.com/lite/?q=words"
|
|
fake = Fake({url: '<html><div class="anomaly-modal"></div><script src="//duckduckgo.com/anomaly.js"></script></html>'})
|
|
c = Collector(config(tmp_path, [{"name": "s", "kind": "search", "value": "words", "everyMinutes": 60}]), fetch=fake)
|
|
c.run_due(now=0.0)
|
|
assert "Warning: collect s failed: SearchBlocked" in lines(capfd)
|
|
assert not list((tmp_path / "collect").glob("s/*.jsonl"))
|
|
|
|
|
|
def test_a_site_is_kept_only_when_it_changed(tmp_path):
|
|
fake = Fake({"https://s.example/": PAGE.format(v=1)})
|
|
c = Collector(config(tmp_path, [{"name": "site", "kind": "site", "value": "https://s.example/", "everyMinutes": 15}]),
|
|
fetch=fake)
|
|
c.run_due(now=0.0)
|
|
c.run_due(now=900.0)
|
|
got = items(tmp_path, "site")
|
|
assert len(got) == 1 and got[0]["title"] == "Status" and got[0]["text"] == "Version 1 is out"
|
|
fake.pages["https://s.example/"] = PAGE.format(v=2)
|
|
c.run_due(now=1800.0)
|
|
assert [i["text"] for i in items(tmp_path, "site")] == ["Version 1 is out", "Version 2 is out"]
|
|
|
|
|
|
def test_a_site_whose_title_alone_changed_is_kept_again(tmp_path):
|
|
page = "<html><head><title>{t}</title></head><body><p>Same body text</p></body></html>"
|
|
fake = Fake({"https://s.example/": page.format(t="Old")})
|
|
c = Collector(config(tmp_path, [{"name": "site", "kind": "site", "value": "https://s.example/", "everyMinutes": 15}]),
|
|
fetch=fake)
|
|
c.run_due(now=0.0)
|
|
fake.pages["https://s.example/"] = page.format(t="New")
|
|
c.run_due(now=900.0)
|
|
assert [i["title"] for i in items(tmp_path, "site")] == ["Old", "New"]
|
|
|
|
|
|
def test_a_target_runs_again_only_after_its_interval_also_after_a_restart(tmp_path):
|
|
fake = Fake({"https://s.example/": PAGE.format(v=1)})
|
|
targets = [{"name": "site", "kind": "site", "value": "https://s.example/", "everyMinutes": 60}]
|
|
Collector(config(tmp_path, targets), fetch=fake).run_due(now=10_000.0)
|
|
again = Collector(config(tmp_path, targets), fetch=fake)
|
|
again.run_due(now=10_000.0 + 59 * 60)
|
|
assert len(fake.asked) == 1
|
|
again.run_due(now=10_000.0 + 60 * 60)
|
|
assert len(fake.asked) == 2
|
|
|
|
|
|
def test_a_failure_counts_as_a_run_and_logs_no_address(tmp_path, capfd):
|
|
fake = Fake({"https://secret-site.example/": ConnectionError("down at https://secret-site.example/")})
|
|
c = Collector(config(tmp_path, [{"name": "x", "kind": "site", "value": "https://secret-site.example/", "everyMinutes": 15}]),
|
|
fetch=fake)
|
|
c.run_due(now=0.0)
|
|
c.run_due(now=60.0)
|
|
assert len(fake.asked) == 1
|
|
out = lines(capfd)
|
|
assert "Warning: collect x failed: ConnectionError" in out
|
|
assert "secret-site" not in out
|
|
|
|
|
|
def test_old_day_files_go_in_every_folder(tmp_path):
|
|
cfg = config(tmp_path, [], keep_days=2)
|
|
today = datetime.now().date()
|
|
for name in ("listed", "removed"):
|
|
d = tmp_path / "collect" / name
|
|
d.mkdir(parents=True)
|
|
for age in (0, 2, 3):
|
|
(d / f"{today - timedelta(days=age)}.jsonl").write_text("{}\n")
|
|
(d / "state.json").write_text("{}")
|
|
Collector(cfg, fetch=Fake({})).run_due(now=time.time())
|
|
for name in ("listed", "removed"):
|
|
left = sorted(p.name for p in (tmp_path / "collect" / name).iterdir())
|
|
assert left == sorted([f"{today - timedelta(days=2)}.jsonl", f"{today}.jsonl", "state.json"])
|
|
|
|
|
|
def test_the_real_fetcher_refuses_addresses_that_are_not_public():
|
|
for url in ("http://127.0.0.1:1/", "http://192.168.0.1/", "file:///etc/passwd", "http://localhost/"):
|
|
with pytest.raises(ValueError):
|
|
safe_get(url)
|
|
|
|
|
|
def test_the_thread_collects_on_its_own_and_stops(tmp_path):
|
|
fake = Fake({"https://s.example/": PAGE.format(v=1)})
|
|
c = Collector(config(tmp_path, [{"name": "site", "kind": "site", "value": "https://s.example/", "everyMinutes": 15}]),
|
|
fetch=fake, check_every=0.05)
|
|
c.start()
|
|
deadline = time.monotonic() + 5
|
|
while not list((tmp_path / "collect").glob("site/*.jsonl")):
|
|
assert time.monotonic() < deadline
|
|
time.sleep(0.02)
|
|
c.stop()
|
|
assert not c.running()
|
|
|
|
|
|
# ---- the real fetcher, against local servers -----------------------------------------------------
|
|
|
|
def serve(handler):
|
|
"""A local HTTP server on 127.0.0.1; returns (port, hits list)."""
|
|
hits = []
|
|
|
|
class H(http.server.BaseHTTPRequestHandler):
|
|
def do_GET(self):
|
|
hits.append(self.path)
|
|
handler(self)
|
|
|
|
def log_message(self, *a):
|
|
pass
|
|
|
|
srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), H)
|
|
srv.daemon_threads = True
|
|
threading.Thread(target=srv.serve_forever, daemon=True).start()
|
|
return srv.server_address[1], hits
|
|
|
|
|
|
def answer(body, status=200, headers=()):
|
|
def h(req):
|
|
req.send_response(status)
|
|
for k, v in headers:
|
|
req.send_header(k, v)
|
|
req.send_header("Content-Length", str(len(body)))
|
|
req.end_headers()
|
|
req.wfile.write(body)
|
|
return h
|
|
|
|
|
|
@pytest.fixture
|
|
def loopback_is_public(monkeypatch):
|
|
"""Lets the fetcher reach the local test servers; every other private address stays refused."""
|
|
real = collect._is_public_ip
|
|
monkeypatch.setattr(collect, "_is_public_ip", lambda ip: ip == ipaddress.ip_address("127.0.0.1") or real(ip))
|
|
|
|
|
|
def test_the_fetcher_follows_redirects_and_checks_every_hop(loopback_is_public):
|
|
def h(req):
|
|
if req.path == "/start":
|
|
answer(b"", 302, [("Location", "/end?x=1")])(req)
|
|
elif req.path == "/private":
|
|
answer(b"", 302, [("Location", "http://10.0.0.1/")])(req)
|
|
else:
|
|
answer(b"the body")(req)
|
|
port, hits = serve(h)
|
|
assert safe_get(f"http://127.0.0.1:{port}/start") == b"the body"
|
|
assert hits == ["/start", "/end?x=1"]
|
|
with pytest.raises(ValueError):
|
|
safe_get(f"http://127.0.0.1:{port}/private")
|
|
|
|
|
|
def test_the_fetcher_connects_to_the_address_it_checked(monkeypatch):
|
|
"""A name that resolves to a public address when checked and to a private one afterwards
|
|
(DNS rebinding) never reaches the private one."""
|
|
port, hits = serve(answer(b"internal"))
|
|
calls = []
|
|
|
|
def resolve(host, *a, **k):
|
|
if host == "rebind.example":
|
|
calls.append(host)
|
|
ip = "1.1.1.1" if len(calls) == 1 else "127.0.0.1"
|
|
else: # an address resolves to itself
|
|
ip = host
|
|
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, port))]
|
|
|
|
monkeypatch.setattr(socket, "getaddrinfo", resolve)
|
|
monkeypatch.setattr(collect, "CONNECT_S", 0.3)
|
|
with pytest.raises(Exception):
|
|
safe_get(f"http://rebind.example:{port}/")
|
|
assert hits == []
|
|
|
|
|
|
def test_the_fetcher_tries_the_next_checked_address_when_one_does_not_connect(loopback_is_public, monkeypatch):
|
|
port, hits = serve(answer(b"from the second address"))
|
|
real = socket.getaddrinfo
|
|
monkeypatch.setattr(socket, "getaddrinfo", lambda host, *a, **k: [
|
|
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("1.1.1.1", port)),
|
|
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", port))] if host == "two.example" else real(host, *a, **k))
|
|
monkeypatch.setattr(collect, "CONNECT_S", 0.3)
|
|
assert safe_get(f"http://two.example:{port}/") == b"from the second address"
|
|
assert hits == ["/"]
|
|
|
|
|
|
def test_the_fetcher_stops_a_server_that_sends_slowly(loopback_is_public, monkeypatch):
|
|
def drip(req):
|
|
req.send_response(200)
|
|
req.send_header("Content-Length", "100000")
|
|
req.end_headers()
|
|
try:
|
|
for _ in range(100):
|
|
req.wfile.write(b"x")
|
|
req.wfile.flush()
|
|
time.sleep(0.1)
|
|
except OSError:
|
|
pass
|
|
port, _ = serve(drip)
|
|
monkeypatch.setattr(collect, "TOTAL_S", 1.0)
|
|
monkeypatch.setattr(collect, "READ_S", 0.5)
|
|
t0 = time.monotonic()
|
|
with pytest.raises(Exception):
|
|
safe_get(f"http://127.0.0.1:{port}/")
|
|
assert time.monotonic() - t0 < 1.5
|
|
|
|
|
|
def test_the_fetcher_never_logs_the_address(monkeypatch, capfd):
|
|
import jarvis.debug
|
|
monkeypatch.setattr(jarvis.debug, "_is_debug_enabled", lambda: True)
|
|
monkeypatch.setattr(socket, "getaddrinfo",
|
|
lambda host, *a, **k: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 80))])
|
|
with pytest.raises(ValueError):
|
|
safe_get("http://secret-host.example/private/path?token=abc123")
|
|
out = capfd.readouterr()
|
|
assert "secret-host" not in out.err + out.out and "abc123" not in out.err + out.out
|
|
|
|
|
|
def test_addresses_with_korean_paths_and_words_are_fetched(loopback_is_public):
|
|
port, hits = serve(answer(b"ok"))
|
|
assert safe_get(f"http://127.0.0.1:{port}/wiki/대한민국?q=한글 검색&x=%2F#top") == b"ok"
|
|
from urllib.parse import quote
|
|
assert hits == [f"/wiki/{quote('대한민국')}?q={quote('한글')}%20{quote('검색')}&x=%2F"]
|
|
|
|
|
|
def test_a_response_cut_short_is_a_failure_not_a_page(loopback_is_public):
|
|
def short(req):
|
|
req.send_response(200)
|
|
req.send_header("Content-Length", "1000")
|
|
req.end_headers()
|
|
req.wfile.write(b"only part")
|
|
req.close_connection = True
|
|
port, _ = serve(short)
|
|
with pytest.raises(Exception):
|
|
safe_get(f"http://127.0.0.1:{port}/")
|
|
|
|
|
|
def test_the_fetcher_stops_a_server_that_sends_its_headers_slowly(loopback_is_public, monkeypatch):
|
|
"""Each byte comes before the per-read limit, so only the overall limit can end it."""
|
|
srv = socket.socket()
|
|
srv.bind(("127.0.0.1", 0))
|
|
srv.listen()
|
|
|
|
def drip():
|
|
conn, _ = srv.accept()
|
|
try:
|
|
for b in b"HTTP/1.1 200 OK\r\nX-Slow: " + b"x" * 1000:
|
|
conn.sendall(bytes([b]))
|
|
time.sleep(0.2)
|
|
except OSError:
|
|
pass
|
|
finally:
|
|
conn.close()
|
|
|
|
threading.Thread(target=drip, daemon=True).start()
|
|
monkeypatch.setattr(collect, "TOTAL_S", 1.0)
|
|
monkeypatch.setattr(collect, "READ_S", 0.5)
|
|
t0 = time.monotonic()
|
|
with pytest.raises(Exception):
|
|
safe_get(f"http://127.0.0.1:{srv.getsockname()[1]}/")
|
|
took = time.monotonic() - t0
|
|
print(f"slow headers: failed after {took:.2f}s")
|
|
assert took < 1.5
|
|
srv.close()
|
|
|
|
|
|
def test_the_fetcher_gives_up_on_a_slow_name_lookup(monkeypatch):
|
|
def slow(host, *a, **k):
|
|
time.sleep(3)
|
|
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("1.1.1.1", 80))]
|
|
|
|
monkeypatch.setattr(socket, "getaddrinfo", slow)
|
|
monkeypatch.setattr(collect, "TOTAL_S", 1.0)
|
|
monkeypatch.setattr(collect, "READ_S", 0.5)
|
|
t0 = time.monotonic()
|
|
with pytest.raises(TimeoutError):
|
|
safe_get("http://slow-dns.example/")
|
|
took = time.monotonic() - t0
|
|
print(f"slow lookup: failed after {took:.2f}s")
|
|
assert took < 1.5
|