fix(app): collection's 15 s limit also covers the name lookup, TLS and headers
The name lookup runs on a daemon thread waited for only until the deadline, and a watchdog shuts the connection down at the deadline, so a server that sends its status line or headers byte by byte within the per-read limit is cut off too. A response cut short is a failure. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -58,84 +58,150 @@ def _is_public_ip(ip) -> bool:
|
||||
or ip.is_multicast or ip.is_unspecified)
|
||||
|
||||
|
||||
def _public_ips(host: str, port: int) -> list:
|
||||
def _resolve(host: str, port: int, deadline: float) -> list:
|
||||
"""getaddrinfo on a daemon thread, waited for only until `deadline` (a late answer is dropped)."""
|
||||
out: dict = {}
|
||||
done = threading.Event()
|
||||
|
||||
def run():
|
||||
try:
|
||||
out["v"] = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
|
||||
except OSError as e:
|
||||
out["e"] = e
|
||||
done.set()
|
||||
|
||||
threading.Thread(target=run, name="collect-dns", daemon=True).start()
|
||||
if not done.wait(max(0.0, deadline - time.monotonic())):
|
||||
raise TimeoutError("took too long")
|
||||
if "e" in out:
|
||||
raise ValueError("name not found")
|
||||
return out["v"]
|
||||
|
||||
|
||||
def _public_ips(host: str, port: int, deadline: float) -> list:
|
||||
"""The addresses to connect to: every address the name resolves to must be public. The caller
|
||||
connects only to these, so a second lookup cannot point elsewhere. Nothing about the address
|
||||
is logged (it is the user's collection target)."""
|
||||
try:
|
||||
addrs = [str(ipaddress.ip_address(host))]
|
||||
except ValueError:
|
||||
try:
|
||||
addrs = list(dict.fromkeys(i[4][0] for i in socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)))
|
||||
except OSError:
|
||||
raise ValueError("name not found") from None
|
||||
addrs = list(dict.fromkeys(i[4][0] for i in _resolve(host, port, deadline)))
|
||||
if not addrs or not all(_is_public_ip(ipaddress.ip_address(a.split("%")[0])) for a in addrs):
|
||||
raise ValueError("not a public address")
|
||||
return addrs
|
||||
|
||||
|
||||
def _open(p, port: int, ips: list, host: str, path: str, deadline: float):
|
||||
"""(pool, response) from the first checked address that connects before `deadline`."""
|
||||
import certifi
|
||||
import urllib3
|
||||
from urllib3.exceptions import ConnectTimeoutError
|
||||
class _Watchdog:
|
||||
"""Shuts the current socket down at the deadline, so a wait anywhere in the request (TLS,
|
||||
status line, headers, body) ends then even if every byte comes within the per-read limit."""
|
||||
|
||||
def __init__(self, deadline: float):
|
||||
self.fired = False
|
||||
self._sock: Optional[socket.socket] = None
|
||||
self._lock = threading.Lock()
|
||||
self._timer = threading.Timer(max(0.0, deadline - time.monotonic()), self._fire)
|
||||
self._timer.daemon = True
|
||||
self._timer.start()
|
||||
|
||||
def _fire(self) -> None:
|
||||
with self._lock:
|
||||
self.fired = True
|
||||
if self._sock is not None:
|
||||
try:
|
||||
self._sock.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def watch(self, sock: socket.socket) -> None:
|
||||
with self._lock:
|
||||
self._sock = sock
|
||||
if self.fired:
|
||||
raise TimeoutError("took too long")
|
||||
|
||||
def cancel(self) -> None:
|
||||
self._timer.cancel()
|
||||
|
||||
|
||||
def _connect(ips: list, port: int, deadline: float) -> socket.socket:
|
||||
"""A socket to the first checked address that connects before `deadline`."""
|
||||
for i, ip in enumerate(ips):
|
||||
left = deadline - time.monotonic()
|
||||
if left <= 0:
|
||||
raise TimeoutError("took too long")
|
||||
timeout = urllib3.Timeout(connect=min(CONNECT_S, left), read=READ_S)
|
||||
if p.scheme == "https":
|
||||
pool = urllib3.HTTPSConnectionPool(ip, port, timeout=timeout, retries=False, cert_reqs="CERT_REQUIRED",
|
||||
ca_certs=certifi.where(), server_hostname=p.hostname,
|
||||
assert_hostname=p.hostname)
|
||||
else:
|
||||
pool = urllib3.HTTPConnectionPool(ip, port, timeout=timeout, retries=False)
|
||||
try:
|
||||
return pool, pool.urlopen("GET", path, headers={"Host": host, "User-Agent": USER_AGENT,
|
||||
"Accept-Encoding": "identity"},
|
||||
redirect=False, retries=False, preload_content=False)
|
||||
except ConnectTimeoutError: # also a refused connection (NewConnectionError)
|
||||
pool.close()
|
||||
return socket.create_connection((ip, port), timeout=min(CONNECT_S, left))
|
||||
except OSError:
|
||||
if i == len(ips) - 1:
|
||||
raise
|
||||
|
||||
|
||||
def _get(url: str, deadline: float, dog: _Watchdog):
|
||||
"""One hop: ("redirect", next url) or ("body", bytes)."""
|
||||
import http.client
|
||||
import ssl
|
||||
|
||||
import certifi
|
||||
|
||||
p = urllib.parse.urlsplit(url)
|
||||
if p.scheme not in ("http", "https") or not p.hostname:
|
||||
raise ValueError("not an http(s) address")
|
||||
port = p.port or (443 if p.scheme == "https" else 80)
|
||||
sock = _connect(_public_ips(p.hostname, port, deadline), port, deadline)
|
||||
try:
|
||||
dog.watch(sock)
|
||||
sock.settimeout(READ_S)
|
||||
if p.scheme == "https":
|
||||
# TLS checks the certificate against the name, not the address connected to. The TLS
|
||||
# socket takes over the connection, so the watchdog watches it from here.
|
||||
ctx = ssl.create_default_context(cafile=certifi.where())
|
||||
conn = http.client.HTTPSConnection(p.hostname, port, context=ctx)
|
||||
sock = ctx.wrap_socket(sock, server_hostname=p.hostname, do_handshake_on_connect=False)
|
||||
dog.watch(sock)
|
||||
sock.do_handshake()
|
||||
else:
|
||||
conn = http.client.HTTPConnection(p.hostname, port)
|
||||
conn.sock = sock
|
||||
path = (p.path or "/") + (f"?{p.query}" if p.query else "")
|
||||
conn.request("GET", path, headers={"User-Agent": USER_AGENT, "Accept-Encoding": "identity"})
|
||||
r = conn.getresponse()
|
||||
if r.status in (301, 302, 303, 307, 308):
|
||||
return "redirect", urllib.parse.urljoin(url, r.getheader("Location", ""))
|
||||
if r.status != 200: # e.g. DuckDuckGo's 202 challenge
|
||||
raise ValueError(f"status {r.status}")
|
||||
body = b""
|
||||
while chunk := r.read1(64 * 1024):
|
||||
body += chunk
|
||||
if len(body) > MAX_BYTES:
|
||||
raise ValueError("response too large")
|
||||
# read1 ends quietly when the watchdog cuts the connection or the server stops early.
|
||||
if dog.fired:
|
||||
raise TimeoutError("took too long")
|
||||
if r.length:
|
||||
raise ValueError("incomplete response")
|
||||
return "body", body
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
|
||||
def safe_get(url: str) -> bytes:
|
||||
"""GET a public http(s) address: every hop's address checked and connected to as checked, at
|
||||
most ``MAX_BYTES`` and ``TOTAL_S`` seconds in all."""
|
||||
# Each socket wait is at most READ_S and none starts after `deadline`, so the whole fetch
|
||||
# (redirects included) ends within TOTAL_S.
|
||||
deadline = time.monotonic() + TOTAL_S - READ_S
|
||||
for _ in range(MAX_REDIRECTS + 1):
|
||||
p = urllib.parse.urlsplit(url)
|
||||
if p.scheme not in ("http", "https") or not p.hostname:
|
||||
raise ValueError("not an http(s) address")
|
||||
port = p.port or (443 if p.scheme == "https" else 80)
|
||||
ips = _public_ips(p.hostname, port)
|
||||
host = p.hostname if p.port is None else f"{p.hostname}:{p.port}"
|
||||
if ":" in p.hostname:
|
||||
host = f"[{p.hostname}]" + ("" if p.port is None else f":{p.port}")
|
||||
path = (p.path or "/") + (f"?{p.query}" if p.query else "")
|
||||
pool, r = _open(p, port, ips, host, path, deadline)
|
||||
try:
|
||||
if r.status in (301, 302, 303, 307, 308):
|
||||
url = urllib.parse.urljoin(url, r.headers.get("Location", ""))
|
||||
continue
|
||||
if r.status != 200: # e.g. DuckDuckGo's 202 challenge
|
||||
raise ValueError(f"status {r.status}")
|
||||
body = b""
|
||||
while chunk := r.read1(64 * 1024):
|
||||
body += chunk
|
||||
if len(body) > MAX_BYTES:
|
||||
raise ValueError("response too large")
|
||||
if time.monotonic() > deadline:
|
||||
raise TimeoutError("took too long")
|
||||
return body
|
||||
finally:
|
||||
r.release_conn()
|
||||
pool.close()
|
||||
raise ValueError("too many redirects")
|
||||
most ``MAX_BYTES`` and ``TOTAL_S`` seconds in all (lookup, connect, TLS, headers and body of
|
||||
every redirect)."""
|
||||
deadline = time.monotonic() + TOTAL_S
|
||||
dog = _Watchdog(deadline)
|
||||
try:
|
||||
for _ in range(MAX_REDIRECTS + 1):
|
||||
kind, value = _get(url, deadline, dog)
|
||||
if kind == "body":
|
||||
return value
|
||||
url = value
|
||||
raise ValueError("too many redirects")
|
||||
except Exception:
|
||||
if dog.fired: # the watchdog cut the connection: whatever broke, it took too long
|
||||
raise TimeoutError("took too long") from None
|
||||
raise
|
||||
finally:
|
||||
dog.cancel()
|
||||
|
||||
|
||||
def _text(html) -> str:
|
||||
|
||||
@@ -335,8 +335,10 @@ still deleted (below).
|
||||
- Every request checks its address like the web search (`web_search.spec.md`): only http(s), and
|
||||
every address the name resolves to must be public; the connection goes only to the checked
|
||||
addresses in their order (TLS still checks the name), so a second lookup cannot lead into the
|
||||
local network. Every redirect is checked the same way. At most 1 MB and 15 s in all (redirects included); a server
|
||||
that sends slowly is cut off. The check logs nothing about the address. The collected text is web
|
||||
local network. Every redirect is checked the same way. At most 1 MB and 15 s in all: the name lookup,
|
||||
connecting, TLS, the headers and the body of every redirect count, and a server that sends
|
||||
slowly, even byte by byte within the per-read limit, is cut off at 15 s. A response cut short is a
|
||||
failure. The check logs nothing about the address. The collected text is web
|
||||
content: it goes only to `<dir>/<name>/<YYYY-MM-DD>.jsonl` (one JSON object per line: `at`, then
|
||||
`title`, `url`, `text` as the kind has them), never to the brain or a log.
|
||||
- Day files older than `keepDays` are deleted in every folder of `dir` at each check, also of
|
||||
|
||||
@@ -319,3 +319,60 @@ def test_the_fetcher_never_logs_the_address(monkeypatch, capfd):
|
||||
safe_get("http://secret-host.example/private/path?token=abc123")
|
||||
out = capfd.readouterr()
|
||||
assert "secret-host" not in out.err + out.out and "abc123" not in out.err + out.out
|
||||
|
||||
|
||||
def test_a_response_cut_short_is_a_failure_not_a_page(loopback_is_public):
|
||||
def short(req):
|
||||
req.send_response(200)
|
||||
req.send_header("Content-Length", "1000")
|
||||
req.end_headers()
|
||||
req.wfile.write(b"only part")
|
||||
req.close_connection = True
|
||||
port, _ = serve(short)
|
||||
with pytest.raises(Exception):
|
||||
safe_get(f"http://127.0.0.1:{port}/")
|
||||
|
||||
|
||||
def test_the_fetcher_stops_a_server_that_sends_its_headers_slowly(loopback_is_public, monkeypatch):
|
||||
"""Each byte comes before the per-read limit, so only the overall limit can end it."""
|
||||
srv = socket.socket()
|
||||
srv.bind(("127.0.0.1", 0))
|
||||
srv.listen()
|
||||
|
||||
def drip():
|
||||
conn, _ = srv.accept()
|
||||
try:
|
||||
for b in b"HTTP/1.1 200 OK\r\nX-Slow: " + b"x" * 1000:
|
||||
conn.sendall(bytes([b]))
|
||||
time.sleep(0.2)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
threading.Thread(target=drip, daemon=True).start()
|
||||
monkeypatch.setattr(collect, "TOTAL_S", 1.0)
|
||||
monkeypatch.setattr(collect, "READ_S", 0.5)
|
||||
t0 = time.monotonic()
|
||||
with pytest.raises(Exception):
|
||||
safe_get(f"http://127.0.0.1:{srv.getsockname()[1]}/")
|
||||
took = time.monotonic() - t0
|
||||
print(f"slow headers: failed after {took:.2f}s")
|
||||
assert took < 1.5
|
||||
srv.close()
|
||||
|
||||
|
||||
def test_the_fetcher_gives_up_on_a_slow_name_lookup(monkeypatch):
|
||||
def slow(host, *a, **k):
|
||||
time.sleep(3)
|
||||
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("1.1.1.1", 80))]
|
||||
|
||||
monkeypatch.setattr(socket, "getaddrinfo", slow)
|
||||
monkeypatch.setattr(collect, "TOTAL_S", 1.0)
|
||||
monkeypatch.setattr(collect, "READ_S", 0.5)
|
||||
t0 = time.monotonic()
|
||||
with pytest.raises(TimeoutError):
|
||||
safe_get("http://slow-dns.example/")
|
||||
took = time.monotonic() - t0
|
||||
print(f"slow lookup: failed after {took:.2f}s")
|
||||
assert took < 1.5
|
||||
|
||||
Reference in New Issue
Block a user