diff --git a/src/voice_worker/collect.py b/src/voice_worker/collect.py index 840d852..d35c486 100644 --- a/src/voice_worker/collect.py +++ b/src/voice_worker/collect.py @@ -135,6 +135,10 @@ def _connect(ips: list, port: int, deadline: float) -> socket.socket: raise +def _ascii(part: str) -> str: + return urllib.parse.quote(part, safe="/%:@!$&'()*+,;=?[]~") + + def _get(url: str, deadline: float, dog: _Watchdog): """One hop: ("redirect", next url) or ("body", bytes).""" import http.client @@ -161,7 +165,9 @@ def _get(url: str, deadline: float, dog: _Watchdog): else: conn = http.client.HTTPConnection(p.hostname, port) conn.sock = sock - path = (p.path or "/") + (f"?{p.query}" if p.query else "") + # The request line is ASCII: other characters (and spaces) go as UTF-8 %HH; existing %HH + # and the reserved separators stay as they are. + path = _ascii(p.path or "/") + (f"?{_ascii(p.query)}" if p.query else "") conn.request("GET", path, headers={"User-Agent": USER_AGENT, "Accept-Encoding": "identity"}) r = conn.getresponse() if r.status in (301, 302, 303, 307, 308): diff --git a/src/voice_worker/voice_worker.spec.md b/src/voice_worker/voice_worker.spec.md index b0a3e37..1790ffc 100644 --- a/src/voice_worker/voice_worker.spec.md +++ b/src/voice_worker/voice_worker.spec.md @@ -338,7 +338,8 @@ still deleted (below). local network. Every redirect is checked the same way. At most 1 MB and 15 s in all: the name lookup, connecting, TLS, the headers and the body of every redirect count, and a server that sends slowly, even byte by byte within the per-read limit, is cut off at 15 s. A response cut short is a - failure. The check logs nothing about the address. The collected text is web + failure. Characters outside ASCII (and spaces) in the path and words go as UTF-8 `%HH`; existing + `%HH` and separators stay. The check logs nothing about the address. The collected text is web content: it goes only to `//.jsonl` (one JSON object per line: `at`, then `title`, `url`, `text` as the kind has them), never to the brain or a log. - Day files older than `keepDays` are deleted in every folder of `dir` at each check, also of diff --git a/tests/test_voice_worker_collect.py b/tests/test_voice_worker_collect.py index 6a704df..7b1d0e3 100644 --- a/tests/test_voice_worker_collect.py +++ b/tests/test_voice_worker_collect.py @@ -321,6 +321,13 @@ def test_the_fetcher_never_logs_the_address(monkeypatch, capfd): assert "secret-host" not in out.err + out.out and "abc123" not in out.err + out.out +def test_addresses_with_korean_paths_and_words_are_fetched(loopback_is_public): + port, hits = serve(answer(b"ok")) + assert safe_get(f"http://127.0.0.1:{port}/wiki/대한민국?q=한글 검색&x=%2F#top") == b"ok" + from urllib.parse import quote + assert hits == [f"/wiki/{quote('대한민국')}?q={quote('한글')}%20{quote('검색')}&x=%2F"] + + def test_a_response_cut_short_is_a_failure_not_a_page(loopback_is_public): def short(req): req.send_response(200)