fix(app): collection's 15 s limit also covers the name lookup, TLS and headers

The name lookup runs on a daemon thread waited for only until the deadline, and a watchdog
shuts the connection down at the deadline, so a server that sends its status line or headers
byte by byte within the per-read limit is cut off too. A response cut short is a failure.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-13 21:46:10 +09:00
parent 9185265010
commit d0cb540fa0
3 changed files with 182 additions and 57 deletions

View File

@@ -58,84 +58,150 @@ def _is_public_ip(ip) -> bool:
or ip.is_multicast or ip.is_unspecified)
def _public_ips(host: str, port: int) -> list:
def _resolve(host: str, port: int, deadline: float) -> list:
"""getaddrinfo on a daemon thread, waited for only until `deadline` (a late answer is dropped)."""
out: dict = {}
done = threading.Event()
def run():
try:
out["v"] = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
except OSError as e:
out["e"] = e
done.set()
threading.Thread(target=run, name="collect-dns", daemon=True).start()
if not done.wait(max(0.0, deadline - time.monotonic())):
raise TimeoutError("took too long")
if "e" in out:
raise ValueError("name not found")
return out["v"]
def _public_ips(host: str, port: int, deadline: float) -> list:
"""The addresses to connect to: every address the name resolves to must be public. The caller
connects only to these, so a second lookup cannot point elsewhere. Nothing about the address
is logged (it is the user's collection target)."""
try:
addrs = [str(ipaddress.ip_address(host))]
except ValueError:
try:
addrs = list(dict.fromkeys(i[4][0] for i in socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)))
except OSError:
raise ValueError("name not found") from None
addrs = list(dict.fromkeys(i[4][0] for i in _resolve(host, port, deadline)))
if not addrs or not all(_is_public_ip(ipaddress.ip_address(a.split("%")[0])) for a in addrs):
raise ValueError("not a public address")
return addrs
def _open(p, port: int, ips: list, host: str, path: str, deadline: float):
"""(pool, response) from the first checked address that connects before `deadline`."""
import certifi
import urllib3
from urllib3.exceptions import ConnectTimeoutError
class _Watchdog:
"""Shuts the current socket down at the deadline, so a wait anywhere in the request (TLS,
status line, headers, body) ends then even if every byte comes within the per-read limit."""
def __init__(self, deadline: float):
self.fired = False
self._sock: Optional[socket.socket] = None
self._lock = threading.Lock()
self._timer = threading.Timer(max(0.0, deadline - time.monotonic()), self._fire)
self._timer.daemon = True
self._timer.start()
def _fire(self) -> None:
with self._lock:
self.fired = True
if self._sock is not None:
try:
self._sock.shutdown(socket.SHUT_RDWR)
except OSError:
pass
def watch(self, sock: socket.socket) -> None:
with self._lock:
self._sock = sock
if self.fired:
raise TimeoutError("took too long")
def cancel(self) -> None:
self._timer.cancel()
def _connect(ips: list, port: int, deadline: float) -> socket.socket:
"""A socket to the first checked address that connects before `deadline`."""
for i, ip in enumerate(ips):
left = deadline - time.monotonic()
if left <= 0:
raise TimeoutError("took too long")
timeout = urllib3.Timeout(connect=min(CONNECT_S, left), read=READ_S)
if p.scheme == "https":
pool = urllib3.HTTPSConnectionPool(ip, port, timeout=timeout, retries=False, cert_reqs="CERT_REQUIRED",
ca_certs=certifi.where(), server_hostname=p.hostname,
assert_hostname=p.hostname)
else:
pool = urllib3.HTTPConnectionPool(ip, port, timeout=timeout, retries=False)
try:
return pool, pool.urlopen("GET", path, headers={"Host": host, "User-Agent": USER_AGENT,
"Accept-Encoding": "identity"},
redirect=False, retries=False, preload_content=False)
except ConnectTimeoutError: # also a refused connection (NewConnectionError)
pool.close()
return socket.create_connection((ip, port), timeout=min(CONNECT_S, left))
except OSError:
if i == len(ips) - 1:
raise
def _get(url: str, deadline: float, dog: _Watchdog):
"""One hop: ("redirect", next url) or ("body", bytes)."""
import http.client
import ssl
import certifi
p = urllib.parse.urlsplit(url)
if p.scheme not in ("http", "https") or not p.hostname:
raise ValueError("not an http(s) address")
port = p.port or (443 if p.scheme == "https" else 80)
sock = _connect(_public_ips(p.hostname, port, deadline), port, deadline)
try:
dog.watch(sock)
sock.settimeout(READ_S)
if p.scheme == "https":
# TLS checks the certificate against the name, not the address connected to. The TLS
# socket takes over the connection, so the watchdog watches it from here.
ctx = ssl.create_default_context(cafile=certifi.where())
conn = http.client.HTTPSConnection(p.hostname, port, context=ctx)
sock = ctx.wrap_socket(sock, server_hostname=p.hostname, do_handshake_on_connect=False)
dog.watch(sock)
sock.do_handshake()
else:
conn = http.client.HTTPConnection(p.hostname, port)
conn.sock = sock
path = (p.path or "/") + (f"?{p.query}" if p.query else "")
conn.request("GET", path, headers={"User-Agent": USER_AGENT, "Accept-Encoding": "identity"})
r = conn.getresponse()
if r.status in (301, 302, 303, 307, 308):
return "redirect", urllib.parse.urljoin(url, r.getheader("Location", ""))
if r.status != 200: # e.g. DuckDuckGo's 202 challenge
raise ValueError(f"status {r.status}")
body = b""
while chunk := r.read1(64 * 1024):
body += chunk
if len(body) > MAX_BYTES:
raise ValueError("response too large")
# read1 ends quietly when the watchdog cuts the connection or the server stops early.
if dog.fired:
raise TimeoutError("took too long")
if r.length:
raise ValueError("incomplete response")
return "body", body
finally:
sock.close()
def safe_get(url: str) -> bytes:
"""GET a public http(s) address: every hop's address checked and connected to as checked, at
most ``MAX_BYTES`` and ``TOTAL_S`` seconds in all."""
# Each socket wait is at most READ_S and none starts after `deadline`, so the whole fetch
# (redirects included) ends within TOTAL_S.
deadline = time.monotonic() + TOTAL_S - READ_S
for _ in range(MAX_REDIRECTS + 1):
p = urllib.parse.urlsplit(url)
if p.scheme not in ("http", "https") or not p.hostname:
raise ValueError("not an http(s) address")
port = p.port or (443 if p.scheme == "https" else 80)
ips = _public_ips(p.hostname, port)
host = p.hostname if p.port is None else f"{p.hostname}:{p.port}"
if ":" in p.hostname:
host = f"[{p.hostname}]" + ("" if p.port is None else f":{p.port}")
path = (p.path or "/") + (f"?{p.query}" if p.query else "")
pool, r = _open(p, port, ips, host, path, deadline)
try:
if r.status in (301, 302, 303, 307, 308):
url = urllib.parse.urljoin(url, r.headers.get("Location", ""))
continue
if r.status != 200: # e.g. DuckDuckGo's 202 challenge
raise ValueError(f"status {r.status}")
body = b""
while chunk := r.read1(64 * 1024):
body += chunk
if len(body) > MAX_BYTES:
raise ValueError("response too large")
if time.monotonic() > deadline:
raise TimeoutError("took too long")
return body
finally:
r.release_conn()
pool.close()
raise ValueError("too many redirects")
most ``MAX_BYTES`` and ``TOTAL_S`` seconds in all (lookup, connect, TLS, headers and body of
every redirect)."""
deadline = time.monotonic() + TOTAL_S
dog = _Watchdog(deadline)
try:
for _ in range(MAX_REDIRECTS + 1):
kind, value = _get(url, deadline, dog)
if kind == "body":
return value
url = value
raise ValueError("too many redirects")
except Exception:
if dog.fired: # the watchdog cut the connection: whatever broke, it took too long
raise TimeoutError("took too long") from None
raise
finally:
dog.cancel()
def _text(html) -> str:

View File

@@ -335,8 +335,10 @@ still deleted (below).
- Every request checks its address like the web search (`web_search.spec.md`): only http(s), and
every address the name resolves to must be public; the connection goes only to the checked
addresses in their order (TLS still checks the name), so a second lookup cannot lead into the
local network. Every redirect is checked the same way. At most 1 MB and 15 s in all (redirects included); a server
that sends slowly is cut off. The check logs nothing about the address. The collected text is web
local network. Every redirect is checked the same way. At most 1 MB and 15 s in all: the name lookup,
connecting, TLS, the headers and the body of every redirect count, and a server that sends
slowly, even byte by byte within the per-read limit, is cut off at 15 s. A response cut short is a
failure. The check logs nothing about the address. The collected text is web
content: it goes only to `<dir>/<name>/<YYYY-MM-DD>.jsonl` (one JSON object per line: `at`, then
`title`, `url`, `text` as the kind has them), never to the brain or a log.
- Day files older than `keepDays` are deleted in every folder of `dir` at each check, also of

View File

@@ -319,3 +319,60 @@ def test_the_fetcher_never_logs_the_address(monkeypatch, capfd):
safe_get("http://secret-host.example/private/path?token=abc123")
out = capfd.readouterr()
assert "secret-host" not in out.err + out.out and "abc123" not in out.err + out.out
def test_a_response_cut_short_is_a_failure_not_a_page(loopback_is_public):
def short(req):
req.send_response(200)
req.send_header("Content-Length", "1000")
req.end_headers()
req.wfile.write(b"only part")
req.close_connection = True
port, _ = serve(short)
with pytest.raises(Exception):
safe_get(f"http://127.0.0.1:{port}/")
def test_the_fetcher_stops_a_server_that_sends_its_headers_slowly(loopback_is_public, monkeypatch):
"""Each byte comes before the per-read limit, so only the overall limit can end it."""
srv = socket.socket()
srv.bind(("127.0.0.1", 0))
srv.listen()
def drip():
conn, _ = srv.accept()
try:
for b in b"HTTP/1.1 200 OK\r\nX-Slow: " + b"x" * 1000:
conn.sendall(bytes([b]))
time.sleep(0.2)
except OSError:
pass
finally:
conn.close()
threading.Thread(target=drip, daemon=True).start()
monkeypatch.setattr(collect, "TOTAL_S", 1.0)
monkeypatch.setattr(collect, "READ_S", 0.5)
t0 = time.monotonic()
with pytest.raises(Exception):
safe_get(f"http://127.0.0.1:{srv.getsockname()[1]}/")
took = time.monotonic() - t0
print(f"slow headers: failed after {took:.2f}s")
assert took < 1.5
srv.close()
def test_the_fetcher_gives_up_on_a_slow_name_lookup(monkeypatch):
def slow(host, *a, **k):
time.sleep(3)
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("1.1.1.1", 80))]
monkeypatch.setattr(socket, "getaddrinfo", slow)
monkeypatch.setattr(collect, "TOTAL_S", 1.0)
monkeypatch.setattr(collect, "READ_S", 0.5)
t0 = time.monotonic()
with pytest.raises(TimeoutError):
safe_get("http://slow-dns.example/")
took = time.monotonic() - t0
print(f"slow lookup: failed after {took:.2f}s")
assert took < 1.5