"""Background collection (voice_worker.spec.md, "Background collection").
The collector runs against a fake fetcher (the real one refuses loopback addresses, which is
checked on its own). Behaviours: each kind keeps only what is new, a target runs again only after
its interval (also across a restart), a failure counts as a run and logs no URL, old day files go,
and nothing collected reaches the log.
"""
from __future__ import annotations
import http.server
import ipaddress
import json
import socket
import threading
import time
from datetime import datetime, timedelta
from pathlib import Path
import pytest
import voice_worker.collect as collect
from voice_worker.collect import Collector, load_config, safe_get
RSS = """
Version {v} is out
" class Fake: """url -> bytes (or an exception to raise); records what was asked.""" def __init__(self, pages): self.pages = pages self.asked = [] def __call__(self, url): self.asked.append(url) v = self.pages[url] if isinstance(v, Exception): raise v return v.encode() if isinstance(v, str) else v def config(tmp_path, targets, keep_days=30): p = tmp_path / "collect.json" p.write_text(json.dumps({"dir": str(tmp_path / "collect"), "keepDays": keep_days, "targets": targets})) return load_config(p) def items(tmp_path, name): out = [] for f in sorted((tmp_path / "collect" / name).glob("*.jsonl")): out += [json.loads(line) for line in f.read_text(encoding="utf-8").splitlines()] return out def lines(capfd): return capfd.readouterr().err def test_rss_keeps_each_new_item_once(tmp_path, capfd): fake = Fake({"https://feed.example/rss": RSS.format(more="")}) cfg = config(tmp_path, [{"name": "뉴스", "kind": "rss", "value": "https://feed.example/rss", "everyMinutes": 15}]) c = Collector(cfg, fetch=fake) c.run_due(now=1000.0) got = items(tmp_path, "뉴스") assert [i["title"] for i in got] == ["First"] assert got[0]["url"] == "https://example.com/1" assert got[0]["text"] == "one bold" assert got[0]["published"] == "Fri, 13 Nov 2026 10:00:00 +0900" fake.pages["https://feed.example/rss"] = RSS.format(more=SECOND) c.run_due(now=1000.0 + 15 * 60) assert [i["title"] for i in items(tmp_path, "뉴스")] == ["First", "Second"] out = lines(capfd) assert "collect 뉴스: 1 new" in out assert "feed.example" not in out and "First" not in out def test_atom_feeds_are_read_too(tmp_path): fake = Fake({"https://a.example/atom": ATOM}) c = Collector(config(tmp_path, [{"name": "a", "kind": "rss", "value": "https://a.example/atom", "everyMinutes": 60}]), fetch=fake) c.run_due(now=0.0) [item] = items(tmp_path, "a") assert (item["title"], item["url"], item["text"]) == ("Atom one", "https://example.org/a1", "sum") def test_search_keeps_new_results_for_the_words(tmp_path): fake = Fake({}) url = "https://lite.duckduckgo.com/lite/?q=rust+release" fake.pages[url] = DDG.format(more="") c = Collector(config(tmp_path, [{"name": "s", "kind": "search", "value": "rust release", "everyMinutes": 60}]), fetch=fake) c.run_due(now=0.0) assert [(i["title"], i["url"]) for i in items(tmp_path, "s")] == [ ("A first result title here", "https://news.example/a"), ("Another result title here", "https://news.example/b")] fake.pages[url] = DDG.format(more='A third result title') c.run_due(now=3600.0) assert [i["url"] for i in items(tmp_path, "s")][-1] == "https://news.example/c" assert len(items(tmp_path, "s")) == 3 def test_a_search_challenge_page_is_a_failure_not_zero_results(tmp_path, capfd): url = "https://lite.duckduckgo.com/lite/?q=words" fake = Fake({url: ''}) c = Collector(config(tmp_path, [{"name": "s", "kind": "search", "value": "words", "everyMinutes": 60}]), fetch=fake) c.run_due(now=0.0) assert "Warning: collect s failed: SearchBlocked" in lines(capfd) assert not list((tmp_path / "collect").glob("s/*.jsonl")) def test_a_site_is_kept_only_when_it_changed(tmp_path): fake = Fake({"https://s.example/": PAGE.format(v=1)}) c = Collector(config(tmp_path, [{"name": "site", "kind": "site", "value": "https://s.example/", "everyMinutes": 15}]), fetch=fake) c.run_due(now=0.0) c.run_due(now=900.0) got = items(tmp_path, "site") assert len(got) == 1 and got[0]["title"] == "Status" and got[0]["text"] == "Version 1 is out" fake.pages["https://s.example/"] = PAGE.format(v=2) c.run_due(now=1800.0) assert [i["text"] for i in items(tmp_path, "site")] == ["Version 1 is out", "Version 2 is out"] def test_a_site_whose_title_alone_changed_is_kept_again(tmp_path): page = "Same body text
" fake = Fake({"https://s.example/": page.format(t="Old")}) c = Collector(config(tmp_path, [{"name": "site", "kind": "site", "value": "https://s.example/", "everyMinutes": 15}]), fetch=fake) c.run_due(now=0.0) fake.pages["https://s.example/"] = page.format(t="New") c.run_due(now=900.0) assert [i["title"] for i in items(tmp_path, "site")] == ["Old", "New"] def test_a_target_runs_again_only_after_its_interval_also_after_a_restart(tmp_path): fake = Fake({"https://s.example/": PAGE.format(v=1)}) targets = [{"name": "site", "kind": "site", "value": "https://s.example/", "everyMinutes": 60}] Collector(config(tmp_path, targets), fetch=fake).run_due(now=10_000.0) again = Collector(config(tmp_path, targets), fetch=fake) again.run_due(now=10_000.0 + 59 * 60) assert len(fake.asked) == 1 again.run_due(now=10_000.0 + 60 * 60) assert len(fake.asked) == 2 def test_a_failure_counts_as_a_run_and_logs_no_address(tmp_path, capfd): fake = Fake({"https://secret-site.example/": ConnectionError("down at https://secret-site.example/")}) c = Collector(config(tmp_path, [{"name": "x", "kind": "site", "value": "https://secret-site.example/", "everyMinutes": 15}]), fetch=fake) c.run_due(now=0.0) c.run_due(now=60.0) assert len(fake.asked) == 1 out = lines(capfd) assert "Warning: collect x failed: ConnectionError" in out assert "secret-site" not in out def test_old_day_files_go_in_every_folder(tmp_path): cfg = config(tmp_path, [], keep_days=2) today = datetime.now().date() for name in ("listed", "removed"): d = tmp_path / "collect" / name d.mkdir(parents=True) for age in (0, 2, 3): (d / f"{today - timedelta(days=age)}.jsonl").write_text("{}\n") (d / "state.json").write_text("{}") Collector(cfg, fetch=Fake({})).run_due(now=time.time()) for name in ("listed", "removed"): left = sorted(p.name for p in (tmp_path / "collect" / name).iterdir()) assert left == sorted([f"{today - timedelta(days=2)}.jsonl", f"{today}.jsonl", "state.json"]) def test_the_real_fetcher_refuses_addresses_that_are_not_public(): for url in ("http://127.0.0.1:1/", "http://192.168.0.1/", "file:///etc/passwd", "http://localhost/"): with pytest.raises(ValueError): safe_get(url) def test_the_thread_collects_on_its_own_and_stops(tmp_path): fake = Fake({"https://s.example/": PAGE.format(v=1)}) c = Collector(config(tmp_path, [{"name": "site", "kind": "site", "value": "https://s.example/", "everyMinutes": 15}]), fetch=fake, check_every=0.05) c.start() deadline = time.monotonic() + 5 while not list((tmp_path / "collect").glob("site/*.jsonl")): assert time.monotonic() < deadline time.sleep(0.02) c.stop() assert not c.running() # ---- the real fetcher, against local servers ----------------------------------------------------- def serve(handler): """A local HTTP server on 127.0.0.1; returns (port, hits list).""" hits = [] class H(http.server.BaseHTTPRequestHandler): def do_GET(self): hits.append(self.path) handler(self) def log_message(self, *a): pass srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), H) srv.daemon_threads = True threading.Thread(target=srv.serve_forever, daemon=True).start() return srv.server_address[1], hits def answer(body, status=200, headers=()): def h(req): req.send_response(status) for k, v in headers: req.send_header(k, v) req.send_header("Content-Length", str(len(body))) req.end_headers() req.wfile.write(body) return h @pytest.fixture def loopback_is_public(monkeypatch): """Lets the fetcher reach the local test servers; every other private address stays refused.""" real = collect._is_public_ip monkeypatch.setattr(collect, "_is_public_ip", lambda ip: ip == ipaddress.ip_address("127.0.0.1") or real(ip)) def test_the_fetcher_follows_redirects_and_checks_every_hop(loopback_is_public): def h(req): if req.path == "/start": answer(b"", 302, [("Location", "/end?x=1")])(req) elif req.path == "/private": answer(b"", 302, [("Location", "http://10.0.0.1/")])(req) else: answer(b"the body")(req) port, hits = serve(h) assert safe_get(f"http://127.0.0.1:{port}/start") == b"the body" assert hits == ["/start", "/end?x=1"] with pytest.raises(ValueError): safe_get(f"http://127.0.0.1:{port}/private") def test_the_fetcher_connects_to_the_address_it_checked(monkeypatch): """A name that resolves to a public address when checked and to a private one afterwards (DNS rebinding) never reaches the private one.""" port, hits = serve(answer(b"internal")) calls = [] def resolve(host, *a, **k): if host == "rebind.example": calls.append(host) ip = "1.1.1.1" if len(calls) == 1 else "127.0.0.1" else: # an address resolves to itself ip = host return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, port))] monkeypatch.setattr(socket, "getaddrinfo", resolve) monkeypatch.setattr(collect, "CONNECT_S", 0.3) with pytest.raises(Exception): safe_get(f"http://rebind.example:{port}/") assert hits == [] def test_the_fetcher_tries_the_next_checked_address_when_one_does_not_connect(loopback_is_public, monkeypatch): port, hits = serve(answer(b"from the second address")) real = socket.getaddrinfo monkeypatch.setattr(socket, "getaddrinfo", lambda host, *a, **k: [ (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("1.1.1.1", port)), (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", port))] if host == "two.example" else real(host, *a, **k)) monkeypatch.setattr(collect, "CONNECT_S", 0.3) assert safe_get(f"http://two.example:{port}/") == b"from the second address" assert hits == ["/"] def test_the_fetcher_stops_a_server_that_sends_slowly(loopback_is_public, monkeypatch): def drip(req): req.send_response(200) req.send_header("Content-Length", "100000") req.end_headers() try: for _ in range(100): req.wfile.write(b"x") req.wfile.flush() time.sleep(0.1) except OSError: pass port, _ = serve(drip) monkeypatch.setattr(collect, "TOTAL_S", 1.0) monkeypatch.setattr(collect, "READ_S", 0.5) t0 = time.monotonic() with pytest.raises(Exception): safe_get(f"http://127.0.0.1:{port}/") assert time.monotonic() - t0 < 1.5 def test_the_fetcher_never_logs_the_address(monkeypatch, capfd): import jarvis.debug monkeypatch.setattr(jarvis.debug, "_is_debug_enabled", lambda: True) monkeypatch.setattr(socket, "getaddrinfo", lambda host, *a, **k: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 80))]) with pytest.raises(ValueError): safe_get("http://secret-host.example/private/path?token=abc123") out = capfd.readouterr() assert "secret-host" not in out.err + out.out and "abc123" not in out.err + out.out def test_addresses_with_korean_paths_and_words_are_fetched(loopback_is_public): port, hits = serve(answer(b"ok")) assert safe_get(f"http://127.0.0.1:{port}/wiki/대한민국?q=한글 검색&x=%2F#top") == b"ok" from urllib.parse import quote assert hits == [f"/wiki/{quote('대한민국')}?q={quote('한글')}%20{quote('검색')}&x=%2F"] def test_a_response_cut_short_is_a_failure_not_a_page(loopback_is_public): def short(req): req.send_response(200) req.send_header("Content-Length", "1000") req.end_headers() req.wfile.write(b"only part") req.close_connection = True port, _ = serve(short) with pytest.raises(Exception): safe_get(f"http://127.0.0.1:{port}/") def test_the_fetcher_stops_a_server_that_sends_its_headers_slowly(loopback_is_public, monkeypatch): """Each byte comes before the per-read limit, so only the overall limit can end it.""" srv = socket.socket() srv.bind(("127.0.0.1", 0)) srv.listen() def drip(): conn, _ = srv.accept() try: for b in b"HTTP/1.1 200 OK\r\nX-Slow: " + b"x" * 1000: conn.sendall(bytes([b])) time.sleep(0.2) except OSError: pass finally: conn.close() threading.Thread(target=drip, daemon=True).start() monkeypatch.setattr(collect, "TOTAL_S", 1.0) monkeypatch.setattr(collect, "READ_S", 0.5) t0 = time.monotonic() with pytest.raises(Exception): safe_get(f"http://127.0.0.1:{srv.getsockname()[1]}/") took = time.monotonic() - t0 print(f"slow headers: failed after {took:.2f}s") assert took < 1.5 srv.close() def test_the_fetcher_gives_up_on_a_slow_name_lookup(monkeypatch): def slow(host, *a, **k): time.sleep(3) return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("1.1.1.1", 80))] monkeypatch.setattr(socket, "getaddrinfo", slow) monkeypatch.setattr(collect, "TOTAL_S", 1.0) monkeypatch.setattr(collect, "READ_S", 0.5) t0 = time.monotonic() with pytest.raises(TimeoutError): safe_get("http://slow-dns.example/") took = time.monotonic() - t0 print(f"slow lookup: failed after {took:.2f}s") assert took < 1.5 def test_only_global_unicast_ipv6_counts_as_public(): for a in ("fec0::1", "::1.2.3.4", "100::1", "2001:2::1", "2002:c0a8::1", "fe80::1", "fd00::1", "::ffff:10.0.0.1"): assert not collect._is_public_ip(ipaddress.ip_address(a)), a for a in ("2606:4700:4700::1111", "2001:4860:4860::8888", "8.8.8.8"): assert collect._is_public_ip(ipaddress.ip_address(a)), a