fix(app): collection connects only to the addresses it checked, within 15 s, logging no address
- the fetcher resolves once, requires every address to be public and connects only to those (TLS still checks the name), so DNS rebinding cannot reach the local network; redirects too - 15 s for the whole fetch also against servers that send slowly; no URL in any log line - a site whose title alone changed is kept again - target names that differ only in case are refused (one Windows folder) - the worker always gets the collection file, so results older than the keep days go also with no targets or the feature off - the results screen lists every target with results, not only those in the newest 100 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -88,13 +88,13 @@ sites, search words and RSS feeds on a schedule; there are **no targets by defau
|
||||
- `collect` (new section): `keepDays` (1-365, default 30) and `targets` (default empty), each
|
||||
`{name, kind, value, everyMinutes, enabled}`. `name` is 1-32 letters, digits, `-` or `_` (any
|
||||
script; it names the target's folder, so not a Windows device name like `con` or `nul`), not
|
||||
listed twice; `kind` is `site`, `search` or `rss`; `value` is an http(s) address for `site` and
|
||||
listed twice, also not in another case (`News` and `news` would share one Windows folder); `kind` is `site`, `search` or `rss`; `value` is an http(s) address for `site` and
|
||||
`rss` (at most 2,000 characters) and the words for `search` (1-200 characters);
|
||||
`everyMinutes` 15-10080 (default 60); `enabled` on by default.
|
||||
- On start, with `features.collect` on and at least one enabled target, the app writes
|
||||
`config/collect.json` (the folder `collect/`, `keepDays` and the enabled targets) and passes it to
|
||||
the worker as `JARVIS_COLLECT_PATH`; otherwise it removes that file and passes nothing. The worker
|
||||
collects while the services run (`voice_worker.spec.md`, "Background collection"); results stay
|
||||
- On every start the app writes `config/collect.json` (the folder `collect/`, `keepDays` and the
|
||||
enabled targets, none with `features.collect` off) and passes it to the worker as
|
||||
`JARVIS_COLLECT_PATH`. Without targets the worker collects nothing but still deletes results
|
||||
older than `keepDays`. The worker collects while the services run (`voice_worker.spec.md`, "Background collection"); results stay
|
||||
in `collect/<name>/<YYYY-MM-DD>.jsonl` in the data folder only.
|
||||
- `results(data, target, limit)` reads them newest first (the newest day file first, its last line
|
||||
first), each item with its `target`; a name that is not a valid target name is refused, so no
|
||||
@@ -305,7 +305,7 @@ back as Korean messages for the screen. The Tauri shell passes the screens' `inv
|
||||
| `settings_fields` / `settings_view` / `settings_apply` `{changes}` | the settings editor above; `restartNeeded` when the services run; `sync` with the chat channels sent to Discord (below) |
|
||||
| `service_start` / `service_stop` / `service_restart` / `dashboard` | the controller and its snapshot; start and restart run in the background, stop also cancels a start |
|
||||
| `discord_channel_info` `{id}` / `channel_sync_status` | a channel's Discord name and topic / each synced chat channel's last result (above) |
|
||||
| `collect_results` `{target, limit}` | collected items newest first (`target` null: every target; `limit` 1-500, default 100) |
|
||||
| `collect_results` `{target, limit}` | `{targets, items}`: every target folder with results, and the collected items newest first (`target` null: every target; `limit` 1-500, default 100) |
|
||||
| `mcp_env_status` / `mcp_env_set` `{server, name, value}` | which MCP environment values are stored (`{server: {NAME: bool}}`, never the values) / store one (an empty value removes it); only for a saved server and name |
|
||||
| `install_status` / `install_start` / `install_cancel` | the install steps and their state, bytes and last line / run what is not done in the background / stop it |
|
||||
| `logs_query` `{tab, level, find, after, limit}` / `logs_error_report` / `logs_export` `{from, to}` | the log hub; the zip goes to `logs/` and its path comes back |
|
||||
@@ -339,7 +339,7 @@ Plain HTML, CSS and an ES module, no build step, dark theme after `DESIGN.md`. A
|
||||
one per line, on/off, delete), and for a saved server a password box per environment variable
|
||||
that stores its value on its own. The 백그라운드 수집 screen has the targets (a table: name, kind,
|
||||
address or words, interval in minutes, on/off, delete), the keep days, and the results (the
|
||||
newest 100, filtered by target, each with its time, title, link and the start of its text shown
|
||||
newest 100, filtered by any target that has results, each with its time, title, link and the start of its text shown
|
||||
as plain text).
|
||||
- **Logs**: tab, level and text filters, pause, live follow (lines after the last seen), "최근 오류
|
||||
복사" (to the clipboard; over 2,000 characters it suggests the zip), and the zip export by days.
|
||||
|
||||
@@ -284,7 +284,9 @@ impl AdminApi {
|
||||
"collect_results" => {
|
||||
let target = args.get("target").and_then(Value::as_str).filter(|t| !t.is_empty());
|
||||
let limit = args.get("limit").and_then(Value::as_u64).unwrap_or(100).clamp(1, 500) as usize;
|
||||
Ok(json!(crate::collect::results(self.dir()?.path(), target, limit)?))
|
||||
let dir = self.dir()?;
|
||||
Ok(json!({"targets": crate::collect::result_targets(dir.path()),
|
||||
"items": crate::collect::results(dir.path(), target, limit)?}))
|
||||
}
|
||||
"channel_sync_status" => Ok(Value::Object(self.syncer()?.status().into_iter()
|
||||
.map(|(id, s)| (id.to_string(), json!(s))).collect())),
|
||||
@@ -409,7 +411,7 @@ mod tests {
|
||||
std::fs::create_dir_all(&day).unwrap();
|
||||
std::fs::write(day.join("2026-11-13.jsonl"), "{\"at\": \"2026-11-13T09:00:00+09:00\", \"title\": \"t\"}\n").unwrap();
|
||||
let r = a.call("collect_results", json!({"target": null})).await.unwrap();
|
||||
assert_eq!(r, json!([{"at": "2026-11-13T09:00:00+09:00", "title": "t", "target": "뉴스"}]));
|
||||
assert_eq!(r, json!({"targets": ["뉴스"], "items": [{"at": "2026-11-13T09:00:00+09:00", "title": "t", "target": "뉴스"}]}));
|
||||
assert!(a.call("collect_results", json!({"target": "../config"})).await.is_err());
|
||||
}
|
||||
|
||||
|
||||
@@ -10,25 +10,19 @@ use crate::settings::{valid_collect_name, Settings};
|
||||
/// The worker's environment variable with the collection file (`voice_worker/collect.py`).
|
||||
pub const COLLECT_ENV_VAR: &str = "JARVIS_COLLECT_PATH";
|
||||
|
||||
/// Write `config/collect.json` with the enabled targets and return it; with the feature off or no
|
||||
/// enabled target, remove it and return None (the worker then collects nothing).
|
||||
pub fn write_config(data: &Path, settings: &Settings) -> std::io::Result<Option<PathBuf>> {
|
||||
/// Write `config/collect.json` and return it. With the feature off it lists no targets: the worker
|
||||
/// then collects nothing but still deletes day files older than the keep days.
|
||||
pub fn write_config(data: &Path, settings: &Settings) -> std::io::Result<PathBuf> {
|
||||
let file = data.join("config").join("collect.json");
|
||||
let targets: Vec<Value> = settings.collect.targets.iter().filter(|t| t.enabled)
|
||||
let targets: Vec<Value> = settings.collect.targets.iter().filter(|t| t.enabled && settings.features.collect)
|
||||
.map(|t| json!({"name": t.name, "kind": t.kind, "value": t.value, "everyMinutes": t.every_minutes}))
|
||||
.collect();
|
||||
if !settings.features.collect || targets.is_empty() {
|
||||
match std::fs::remove_file(&file) {
|
||||
Err(e) if e.kind() != std::io::ErrorKind::NotFound => return Err(e),
|
||||
_ => return Ok(None),
|
||||
}
|
||||
}
|
||||
let cfg = json!({"dir": data.join("collect").to_string_lossy(), "keepDays": settings.collect.keep_days,
|
||||
"targets": targets});
|
||||
let mut text = serde_json::to_string_pretty(&cfg).unwrap();
|
||||
text.push('\n');
|
||||
crate::datadir::write_atomic(&file, text.as_bytes())?;
|
||||
Ok(Some(file))
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
/// The day files of one target folder, newest first.
|
||||
@@ -41,17 +35,25 @@ fn day_files(folder: &Path) -> Vec<PathBuf> {
|
||||
files
|
||||
}
|
||||
|
||||
/// The target folders that hold results, sorted.
|
||||
pub fn result_targets(data: &Path) -> Vec<String> {
|
||||
let root = data.join("collect");
|
||||
let mut names: Vec<String> = std::fs::read_dir(&root).into_iter().flatten().flatten()
|
||||
.filter(|e| e.path().is_dir())
|
||||
.filter_map(|e| e.file_name().to_str().map(str::to_owned))
|
||||
.filter(|n| valid_collect_name(n) && !day_files(&root.join(n)).is_empty())
|
||||
.collect();
|
||||
names.sort();
|
||||
names
|
||||
}
|
||||
|
||||
/// Collected items newest first, each with its `target`; `target` None reads every target folder.
|
||||
pub fn results(data: &Path, target: Option<&str>, limit: usize) -> Result<Vec<Value>, String> {
|
||||
let root = data.join("collect");
|
||||
let names: Vec<String> = match target {
|
||||
Some(t) if !valid_collect_name(t) => return Err(format!("not a collection target name: {t}")),
|
||||
Some(t) => vec![t.to_owned()],
|
||||
None => std::fs::read_dir(&root).into_iter().flatten().flatten()
|
||||
.filter(|e| e.path().is_dir())
|
||||
.filter_map(|e| e.file_name().to_str().map(str::to_owned))
|
||||
.filter(|n| valid_collect_name(n))
|
||||
.collect(),
|
||||
None => result_targets(data),
|
||||
};
|
||||
let mut out: Vec<Value> = Vec::new();
|
||||
for name in names {
|
||||
@@ -87,23 +89,21 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_worker_gets_only_enabled_targets_and_nothing_without_them() {
|
||||
fn the_worker_gets_only_enabled_targets_and_the_keep_days_also_without_targets() {
|
||||
let data = temp_dir("collect-cfg");
|
||||
let s = settings(json!({"collect": {"keepDays": 3, "targets": [
|
||||
{"name": "a", "kind": "rss", "value": "https://a.example/rss", "everyMinutes": 30},
|
||||
{"name": "b", "kind": "site", "value": "https://b.example", "enabled": false}]}}));
|
||||
let file = write_config(&data, &s).unwrap().unwrap();
|
||||
let cfg: Value = serde_json::from_str(&std::fs::read_to_string(&file).unwrap()).unwrap();
|
||||
assert_eq!(cfg, json!({"dir": data.join("collect").to_string_lossy(), "keepDays": 3,
|
||||
let read = |f: &Path| serde_json::from_str::<Value>(&std::fs::read_to_string(f).unwrap()).unwrap();
|
||||
let dir = data.join("collect").to_string_lossy().into_owned();
|
||||
let file = write_config(&data, &s).unwrap();
|
||||
assert_eq!(read(&file), json!({"dir": dir, "keepDays": 3,
|
||||
"targets": [{"name": "a", "kind": "rss", "value": "https://a.example/rss", "everyMinutes": 30}]}));
|
||||
|
||||
// Feature off or no targets: nothing to collect, but old results still go after keepDays.
|
||||
let mut off = s.clone();
|
||||
off.features.collect = false;
|
||||
assert_eq!(write_config(&data, &off).unwrap(), None);
|
||||
assert!(!file.exists());
|
||||
write_config(&data, &s).unwrap();
|
||||
assert_eq!(write_config(&data, &settings(json!({}))).unwrap(), None);
|
||||
assert!(!file.exists());
|
||||
assert_eq!(read(&write_config(&data, &off).unwrap()), json!({"dir": dir, "keepDays": 3, "targets": []}));
|
||||
assert_eq!(read(&write_config(&data, &settings(json!({}))).unwrap()), json!({"dir": dir, "keepDays": 30, "targets": []}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -125,6 +125,9 @@ mod tests {
|
||||
assert_eq!(titles(results(&data, Some("뉴스"), 2).unwrap()), ["뉴스:n2", "뉴스:n1"]);
|
||||
assert_eq!(titles(results(&data, None, 2).unwrap()), ["뉴스:n2", "site:s1"]);
|
||||
assert!(results(&data, Some("missing"), 10).unwrap().is_empty());
|
||||
// Every target with results is listed, also one whose items are all older than the newest.
|
||||
std::fs::create_dir_all(data.join("collect").join("empty")).unwrap();
|
||||
assert_eq!(result_targets(&data), ["site", "뉴스"]);
|
||||
assert!(results(&data, Some(".."), 10).is_err());
|
||||
assert!(results(&data, Some("../config"), 10).is_err());
|
||||
}
|
||||
|
||||
@@ -70,10 +70,9 @@ pub async fn start_services(data_dir: &Path, llm: Arc<Llm>) -> Result<Services,
|
||||
}
|
||||
wopts.env.insert("JARVIS_CONFIG_PATH".into(), brain_config.to_string_lossy().into_owned());
|
||||
wopts.env.insert(crate::features::MCP_ENV_VAR.into(), mcp_env.to_string());
|
||||
// Background collection: only with the feature on and enabled targets (admin.spec.md).
|
||||
// Background collection (admin.spec.md): the enabled targets, and the keep days in any case.
|
||||
match crate::collect::write_config(data_dir, &settings) {
|
||||
Ok(Some(f)) => { wopts.env.insert(crate::collect::COLLECT_ENV_VAR.into(), f.to_string_lossy().into_owned()); }
|
||||
Ok(None) => {}
|
||||
Ok(f) => { wopts.env.insert(crate::collect::COLLECT_ENV_VAR.into(), f.to_string_lossy().into_owned()); }
|
||||
Err(e) => return Err(after_llm_stop(&llm, format!("could not write the collection settings: {e}")).await.into()),
|
||||
}
|
||||
let worker_process = match WorkerProcess::new(wopts) {
|
||||
|
||||
@@ -550,8 +550,9 @@ fn parse_collect(raw: &Map<String, Value>, d: &CollectSettings) -> Result<Collec
|
||||
if !valid_collect_name(&name) {
|
||||
return err(format!("{path}.name must be 1-32 letters, digits, - or _ (not a device name like con)"));
|
||||
}
|
||||
if targets.iter().any(|t| t.name == name) {
|
||||
return err(format!("{path}.name is listed twice"));
|
||||
// Windows folder names ignore case: `News` and `news` would share one folder and state.
|
||||
if targets.iter().any(|t| t.name.to_lowercase() == name.to_lowercase()) {
|
||||
return err(format!("{path}.name is listed twice (names that differ only in case count as the same)"));
|
||||
}
|
||||
let kind = get_str(Some(m), "kind", &path, "")?;
|
||||
let value = get_str(Some(m), "value", &path, "")?.trim().to_owned();
|
||||
@@ -867,6 +868,8 @@ mod tests {
|
||||
(t(json!({"everyMinutes": 10081})), "collect.targets[0].everyMinutes"),
|
||||
(json!({"collect": {"targets": [{"name": "a", "kind": "rss", "value": "https://a"},
|
||||
{"name": "a", "kind": "rss", "value": "https://b"}]}}), "collect.targets[1].name"),
|
||||
(json!({"collect": {"targets": [{"name": "News", "kind": "rss", "value": "https://a"},
|
||||
{"name": "news", "kind": "rss", "value": "https://b"}]}}), "collect.targets[1].name"),
|
||||
];
|
||||
for (raw, key) in cases {
|
||||
let e = parse_settings(&raw).unwrap_err();
|
||||
|
||||
@@ -286,11 +286,10 @@ async function collectResults() {
|
||||
const list = h("div");
|
||||
const load = async () => {
|
||||
try {
|
||||
const items = await api("collect_results", { target: pick.value || null, limit: 100 });
|
||||
if (!pick.value) {
|
||||
for (const name of new Set(items.map((i) => i.target)))
|
||||
if (![...pick.options].some((o) => o.value === name)) pick.append(h("option", { value: name }, name));
|
||||
}
|
||||
const r = await api("collect_results", { target: pick.value || null, limit: 100 });
|
||||
const items = r.items;
|
||||
for (const name of r.targets)
|
||||
if (![...pick.options].some((o) => o.value === name)) pick.append(h("option", { value: name }, name));
|
||||
list.replaceChildren(items.length ? h("table", {}, h("tbody", {}, items.map((i) => h("tr", {},
|
||||
h("td", { class: "small muted" }, (i.at || "").replace("T", " ").slice(0, 16)),
|
||||
h("td", { class: "small" }, i.target),
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -4,8 +4,10 @@ collection"). Results go only to day files in the data folder; logs carry target
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import json
|
||||
import re
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
@@ -18,7 +20,9 @@ from typing import Callable, Optional
|
||||
from .log import debug_log, notice
|
||||
|
||||
MAX_BYTES = 1024 * 1024
|
||||
TIMEOUT_S = 15
|
||||
TOTAL_S = 15.0
|
||||
CONNECT_S = 10.0
|
||||
READ_S = 5.0
|
||||
MAX_REDIRECTS = 5
|
||||
SITE_CHARS = 20_000
|
||||
SUMMARY_CHARS = 2_000
|
||||
@@ -49,28 +53,88 @@ def load_config(path) -> Config:
|
||||
[Target(t["name"], t["kind"], t["value"], float(t["everyMinutes"]) * 60) for t in raw.get("targets", [])])
|
||||
|
||||
|
||||
def safe_get(url: str) -> bytes:
|
||||
"""GET a public http(s) address: every redirect checked again, at most ``MAX_BYTES``."""
|
||||
import requests
|
||||
from jarvis.tools.builtin.web_search import _is_public_url
|
||||
def _is_public_ip(ip) -> bool:
|
||||
return not (ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved
|
||||
or ip.is_multicast or ip.is_unspecified)
|
||||
|
||||
|
||||
def _public_ips(host: str, port: int) -> list:
|
||||
"""The addresses to connect to: every address the name resolves to must be public. The caller
|
||||
connects only to these, so a second lookup cannot point elsewhere. Nothing about the address
|
||||
is logged (it is the user's collection target)."""
|
||||
try:
|
||||
addrs = [str(ipaddress.ip_address(host))]
|
||||
except ValueError:
|
||||
try:
|
||||
addrs = list(dict.fromkeys(i[4][0] for i in socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)))
|
||||
except OSError:
|
||||
raise ValueError("name not found") from None
|
||||
if not addrs or not all(_is_public_ip(ipaddress.ip_address(a.split("%")[0])) for a in addrs):
|
||||
raise ValueError("not a public address")
|
||||
return addrs
|
||||
|
||||
|
||||
def _open(p, port: int, ips: list, host: str, path: str, deadline: float):
|
||||
"""(pool, response) from the first checked address that connects before `deadline`."""
|
||||
import certifi
|
||||
import urllib3
|
||||
from urllib3.exceptions import ConnectTimeoutError
|
||||
|
||||
for i, ip in enumerate(ips):
|
||||
left = deadline - time.monotonic()
|
||||
if left <= 0:
|
||||
raise TimeoutError("took too long")
|
||||
timeout = urllib3.Timeout(connect=min(CONNECT_S, left), read=READ_S)
|
||||
if p.scheme == "https":
|
||||
pool = urllib3.HTTPSConnectionPool(ip, port, timeout=timeout, retries=False, cert_reqs="CERT_REQUIRED",
|
||||
ca_certs=certifi.where(), server_hostname=p.hostname,
|
||||
assert_hostname=p.hostname)
|
||||
else:
|
||||
pool = urllib3.HTTPConnectionPool(ip, port, timeout=timeout, retries=False)
|
||||
try:
|
||||
return pool, pool.urlopen("GET", path, headers={"Host": host, "User-Agent": USER_AGENT,
|
||||
"Accept-Encoding": "identity"},
|
||||
redirect=False, retries=False, preload_content=False)
|
||||
except ConnectTimeoutError: # also a refused connection (NewConnectionError)
|
||||
pool.close()
|
||||
if i == len(ips) - 1:
|
||||
raise
|
||||
|
||||
|
||||
def safe_get(url: str) -> bytes:
|
||||
"""GET a public http(s) address: every hop's address checked and connected to as checked, at
|
||||
most ``MAX_BYTES`` and ``TOTAL_S`` seconds in all."""
|
||||
# Each socket wait is at most READ_S and none starts after `deadline`, so the whole fetch
|
||||
# (redirects included) ends within TOTAL_S.
|
||||
deadline = time.monotonic() + TOTAL_S - READ_S
|
||||
for _ in range(MAX_REDIRECTS + 1):
|
||||
if not _is_public_url(url):
|
||||
raise ValueError("not a public address")
|
||||
with requests.get(url, headers={"User-Agent": USER_AGENT}, timeout=TIMEOUT_S,
|
||||
allow_redirects=False, stream=True) as r:
|
||||
if r.is_redirect or r.is_permanent_redirect:
|
||||
p = urllib.parse.urlsplit(url)
|
||||
if p.scheme not in ("http", "https") or not p.hostname:
|
||||
raise ValueError("not an http(s) address")
|
||||
port = p.port or (443 if p.scheme == "https" else 80)
|
||||
ips = _public_ips(p.hostname, port)
|
||||
host = p.hostname if p.port is None else f"{p.hostname}:{p.port}"
|
||||
if ":" in p.hostname:
|
||||
host = f"[{p.hostname}]" + ("" if p.port is None else f":{p.port}")
|
||||
path = (p.path or "/") + (f"?{p.query}" if p.query else "")
|
||||
pool, r = _open(p, port, ips, host, path, deadline)
|
||||
try:
|
||||
if r.status in (301, 302, 303, 307, 308):
|
||||
url = urllib.parse.urljoin(url, r.headers.get("Location", ""))
|
||||
continue
|
||||
r.raise_for_status()
|
||||
if r.status_code != 200: # e.g. DuckDuckGo's 202 challenge
|
||||
raise ValueError(f"status {r.status_code}")
|
||||
if r.status != 200: # e.g. DuckDuckGo's 202 challenge
|
||||
raise ValueError(f"status {r.status}")
|
||||
body = b""
|
||||
for chunk in r.iter_content(64 * 1024):
|
||||
while chunk := r.read1(64 * 1024):
|
||||
body += chunk
|
||||
if len(body) > MAX_BYTES:
|
||||
raise ValueError("response too large")
|
||||
if time.monotonic() > deadline:
|
||||
raise TimeoutError("took too long")
|
||||
return body
|
||||
finally:
|
||||
r.release_conn()
|
||||
pool.close()
|
||||
raise ValueError("too many redirects")
|
||||
|
||||
|
||||
@@ -186,7 +250,7 @@ class Collector:
|
||||
def _run(self, t: Target, state: dict) -> int:
|
||||
if t.kind == "site":
|
||||
[page] = _site(self.fetch(t.value))
|
||||
digest = hashlib.sha256(page["text"].encode()).hexdigest()
|
||||
digest = hashlib.sha256(json.dumps([page["title"], page["text"]]).encode()).hexdigest()
|
||||
if digest == state.get("hash"):
|
||||
return 0
|
||||
state["hash"] = digest
|
||||
|
||||
@@ -322,17 +322,21 @@ which carry Discord's own time.
|
||||
With `JARVIS_COLLECT_PATH` set, the worker collects on its own thread, apart from the voice and the
|
||||
brain (`app/src/admin.spec.md`, "Background collection"). The file is
|
||||
`{"dir": "<data>/collect", "keepDays": 30, "targets": [{"name", "kind", "value", "everyMinutes"}]}`;
|
||||
the app writes only enabled targets. Nothing is collected without targets.
|
||||
the app writes only enabled targets. Nothing is collected without targets, but old day files are
|
||||
still deleted (below).
|
||||
|
||||
- A target runs when `everyMinutes` passed since its last run (also across restarts; the last run is
|
||||
kept in `<dir>/<name>/state.json`), one target at a time, checked every 30 s. A failed run counts
|
||||
as a run, so a broken site is tried again only at its next interval.
|
||||
- Kinds: `site` keeps the page's title and text (at most 20,000 characters) when it differs from the
|
||||
last kept one; `rss` (RSS or Atom) keeps each new item (title, link, time, summary of at most 2,000
|
||||
- Kinds: `site` keeps the page's title and text (at most 20,000 characters) when either differs from
|
||||
the last kept ones; `rss` (RSS or Atom) keeps each new item (title, link, time, summary of at most 2,000
|
||||
characters, new by its id or link); `search` keeps each new web search result (title and link) for
|
||||
the words, from DuckDuckGo's lite page, which needs no key (its bot challenge is a failed run, not "no results").
|
||||
- Every request goes through the web search's address check (`web_search.spec.md`): only public
|
||||
http(s) addresses, every redirect checked again, at most 1 MB, 15 s. The collected text is web
|
||||
- Every request checks its address like the web search (`web_search.spec.md`): only http(s), and
|
||||
every address the name resolves to must be public; the connection goes only to the checked
|
||||
addresses in their order (TLS still checks the name), so a second lookup cannot lead into the
|
||||
local network. Every redirect is checked the same way. At most 1 MB and 15 s in all (redirects included); a server
|
||||
that sends slowly is cut off. The check logs nothing about the address. The collected text is web
|
||||
content: it goes only to `<dir>/<name>/<YYYY-MM-DD>.jsonl` (one JSON object per line: `at`, then
|
||||
`title`, `url`, `text` as the kind has them), never to the brain or a log.
|
||||
- Day files older than `keepDays` are deleted in every folder of `dir` at each check, also of
|
||||
|
||||
@@ -8,13 +8,18 @@ and nothing collected reaches the log.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import http.server
|
||||
import ipaddress
|
||||
import json
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import voice_worker.collect as collect
|
||||
from voice_worker.collect import Collector, load_config, safe_get
|
||||
|
||||
RSS = """<?xml version="1.0"?><rss version="2.0"><channel><title>feed</title>
|
||||
@@ -131,6 +136,17 @@ def test_a_site_is_kept_only_when_it_changed(tmp_path):
|
||||
assert [i["text"] for i in items(tmp_path, "site")] == ["Version 1 is out", "Version 2 is out"]
|
||||
|
||||
|
||||
def test_a_site_whose_title_alone_changed_is_kept_again(tmp_path):
|
||||
page = "<html><head><title>{t}</title></head><body><p>Same body text</p></body></html>"
|
||||
fake = Fake({"https://s.example/": page.format(t="Old")})
|
||||
c = Collector(config(tmp_path, [{"name": "site", "kind": "site", "value": "https://s.example/", "everyMinutes": 15}]),
|
||||
fetch=fake)
|
||||
c.run_due(now=0.0)
|
||||
fake.pages["https://s.example/"] = page.format(t="New")
|
||||
c.run_due(now=900.0)
|
||||
assert [i["title"] for i in items(tmp_path, "site")] == ["Old", "New"]
|
||||
|
||||
|
||||
def test_a_target_runs_again_only_after_its_interval_also_after_a_restart(tmp_path):
|
||||
fake = Fake({"https://s.example/": PAGE.format(v=1)})
|
||||
targets = [{"name": "site", "kind": "site", "value": "https://s.example/", "everyMinutes": 60}]
|
||||
@@ -186,3 +202,120 @@ def test_the_thread_collects_on_its_own_and_stops(tmp_path):
|
||||
time.sleep(0.02)
|
||||
c.stop()
|
||||
assert not c.running()
|
||||
|
||||
|
||||
# ---- the real fetcher, against local servers -----------------------------------------------------
|
||||
|
||||
def serve(handler):
|
||||
"""A local HTTP server on 127.0.0.1; returns (port, hits list)."""
|
||||
hits = []
|
||||
|
||||
class H(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
hits.append(self.path)
|
||||
handler(self)
|
||||
|
||||
def log_message(self, *a):
|
||||
pass
|
||||
|
||||
srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), H)
|
||||
srv.daemon_threads = True
|
||||
threading.Thread(target=srv.serve_forever, daemon=True).start()
|
||||
return srv.server_address[1], hits
|
||||
|
||||
|
||||
def answer(body, status=200, headers=()):
|
||||
def h(req):
|
||||
req.send_response(status)
|
||||
for k, v in headers:
|
||||
req.send_header(k, v)
|
||||
req.send_header("Content-Length", str(len(body)))
|
||||
req.end_headers()
|
||||
req.wfile.write(body)
|
||||
return h
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def loopback_is_public(monkeypatch):
|
||||
"""Lets the fetcher reach the local test servers; every other private address stays refused."""
|
||||
real = collect._is_public_ip
|
||||
monkeypatch.setattr(collect, "_is_public_ip", lambda ip: ip == ipaddress.ip_address("127.0.0.1") or real(ip))
|
||||
|
||||
|
||||
def test_the_fetcher_follows_redirects_and_checks_every_hop(loopback_is_public):
|
||||
def h(req):
|
||||
if req.path == "/start":
|
||||
answer(b"", 302, [("Location", "/end?x=1")])(req)
|
||||
elif req.path == "/private":
|
||||
answer(b"", 302, [("Location", "http://10.0.0.1/")])(req)
|
||||
else:
|
||||
answer(b"the body")(req)
|
||||
port, hits = serve(h)
|
||||
assert safe_get(f"http://127.0.0.1:{port}/start") == b"the body"
|
||||
assert hits == ["/start", "/end?x=1"]
|
||||
with pytest.raises(ValueError):
|
||||
safe_get(f"http://127.0.0.1:{port}/private")
|
||||
|
||||
|
||||
def test_the_fetcher_connects_to_the_address_it_checked(monkeypatch):
|
||||
"""A name that resolves to a public address when checked and to a private one afterwards
|
||||
(DNS rebinding) never reaches the private one."""
|
||||
port, hits = serve(answer(b"internal"))
|
||||
calls = []
|
||||
|
||||
def resolve(host, *a, **k):
|
||||
if host == "rebind.example":
|
||||
calls.append(host)
|
||||
ip = "1.1.1.1" if len(calls) == 1 else "127.0.0.1"
|
||||
else: # an address resolves to itself
|
||||
ip = host
|
||||
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, port))]
|
||||
|
||||
monkeypatch.setattr(socket, "getaddrinfo", resolve)
|
||||
monkeypatch.setattr(collect, "CONNECT_S", 0.3)
|
||||
with pytest.raises(Exception):
|
||||
safe_get(f"http://rebind.example:{port}/")
|
||||
assert hits == []
|
||||
|
||||
|
||||
def test_the_fetcher_tries_the_next_checked_address_when_one_does_not_connect(loopback_is_public, monkeypatch):
|
||||
port, hits = serve(answer(b"from the second address"))
|
||||
real = socket.getaddrinfo
|
||||
monkeypatch.setattr(socket, "getaddrinfo", lambda host, *a, **k: [
|
||||
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("1.1.1.1", port)),
|
||||
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", port))] if host == "two.example" else real(host, *a, **k))
|
||||
monkeypatch.setattr(collect, "CONNECT_S", 0.3)
|
||||
assert safe_get(f"http://two.example:{port}/") == b"from the second address"
|
||||
assert hits == ["/"]
|
||||
|
||||
|
||||
def test_the_fetcher_stops_a_server_that_sends_slowly(loopback_is_public, monkeypatch):
|
||||
def drip(req):
|
||||
req.send_response(200)
|
||||
req.send_header("Content-Length", "100000")
|
||||
req.end_headers()
|
||||
try:
|
||||
for _ in range(100):
|
||||
req.wfile.write(b"x")
|
||||
req.wfile.flush()
|
||||
time.sleep(0.1)
|
||||
except OSError:
|
||||
pass
|
||||
port, _ = serve(drip)
|
||||
monkeypatch.setattr(collect, "TOTAL_S", 1.0)
|
||||
monkeypatch.setattr(collect, "READ_S", 0.5)
|
||||
t0 = time.monotonic()
|
||||
with pytest.raises(Exception):
|
||||
safe_get(f"http://127.0.0.1:{port}/")
|
||||
assert time.monotonic() - t0 < 1.5
|
||||
|
||||
|
||||
def test_the_fetcher_never_logs_the_address(monkeypatch, capfd):
|
||||
import jarvis.debug
|
||||
monkeypatch.setattr(jarvis.debug, "_is_debug_enabled", lambda: True)
|
||||
monkeypatch.setattr(socket, "getaddrinfo",
|
||||
lambda host, *a, **k: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 80))])
|
||||
with pytest.raises(ValueError):
|
||||
safe_get("http://secret-host.example/private/path?token=abc123")
|
||||
out = capfd.readouterr()
|
||||
assert "secret-host" not in out.err + out.out and "abc123" not in out.err + out.out
|
||||
|
||||
Reference in New Issue
Block a user