- the fetcher resolves once, requires every address to be public and connects only to those (TLS still checks the name), so DNS rebinding cannot reach the local network; redirects too - 15 s for the whole fetch also against servers that send slowly; no URL in any log line - a site whose title alone changed is kept again - target names that differ only in case are refused (one Windows folder) - the worker always gets the collection file, so results older than the keep days go also with no targets or the feature off - the results screen lists every target with results, not only those in the newest 100 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
135 lines
6.6 KiB
Rust
135 lines
6.6 KiB
Rust
//! Background collection (`app/src/admin.spec.md`, "Background collection"): the worker's
|
|
//! collection file, and the collected items for the 백그라운드 수집 screen.
|
|
|
|
use std::path::{Path, PathBuf};
|
|
|
|
use serde_json::{json, Value};
|
|
|
|
use crate::settings::{valid_collect_name, Settings};
|
|
|
|
/// The worker's environment variable with the collection file (`voice_worker/collect.py`).
|
|
pub const COLLECT_ENV_VAR: &str = "JARVIS_COLLECT_PATH";
|
|
|
|
/// Write `config/collect.json` and return it. With the feature off it lists no targets: the worker
|
|
/// then collects nothing but still deletes day files older than the keep days.
|
|
pub fn write_config(data: &Path, settings: &Settings) -> std::io::Result<PathBuf> {
|
|
let file = data.join("config").join("collect.json");
|
|
let targets: Vec<Value> = settings.collect.targets.iter().filter(|t| t.enabled && settings.features.collect)
|
|
.map(|t| json!({"name": t.name, "kind": t.kind, "value": t.value, "everyMinutes": t.every_minutes}))
|
|
.collect();
|
|
let cfg = json!({"dir": data.join("collect").to_string_lossy(), "keepDays": settings.collect.keep_days,
|
|
"targets": targets});
|
|
let mut text = serde_json::to_string_pretty(&cfg).unwrap();
|
|
text.push('\n');
|
|
crate::datadir::write_atomic(&file, text.as_bytes())?;
|
|
Ok(file)
|
|
}
|
|
|
|
/// The day files of one target folder, newest first.
|
|
fn day_files(folder: &Path) -> Vec<PathBuf> {
|
|
let mut files: Vec<PathBuf> = std::fs::read_dir(folder).into_iter().flatten().flatten().map(|e| e.path())
|
|
.filter(|p| p.file_name().and_then(|n| n.to_str()).is_some_and(|n| n.len() == 16 && n.ends_with(".jsonl")))
|
|
.collect();
|
|
files.sort();
|
|
files.reverse();
|
|
files
|
|
}
|
|
|
|
/// The target folders that hold results, sorted.
|
|
pub fn result_targets(data: &Path) -> Vec<String> {
|
|
let root = data.join("collect");
|
|
let mut names: Vec<String> = std::fs::read_dir(&root).into_iter().flatten().flatten()
|
|
.filter(|e| e.path().is_dir())
|
|
.filter_map(|e| e.file_name().to_str().map(str::to_owned))
|
|
.filter(|n| valid_collect_name(n) && !day_files(&root.join(n)).is_empty())
|
|
.collect();
|
|
names.sort();
|
|
names
|
|
}
|
|
|
|
/// Collected items newest first, each with its `target`; `target` None reads every target folder.
|
|
pub fn results(data: &Path, target: Option<&str>, limit: usize) -> Result<Vec<Value>, String> {
|
|
let root = data.join("collect");
|
|
let names: Vec<String> = match target {
|
|
Some(t) if !valid_collect_name(t) => return Err(format!("not a collection target name: {t}")),
|
|
Some(t) => vec![t.to_owned()],
|
|
None => result_targets(data),
|
|
};
|
|
let mut out: Vec<Value> = Vec::new();
|
|
for name in names {
|
|
let mut n = 0;
|
|
'files: for f in day_files(&root.join(&name)) {
|
|
let text = std::fs::read_to_string(&f).unwrap_or_default();
|
|
for line in text.lines().rev() {
|
|
if let Ok(Value::Object(mut item)) = serde_json::from_str::<Value>(line) {
|
|
item.insert("target".into(), Value::String(name.clone()));
|
|
out.push(Value::Object(item));
|
|
n += 1;
|
|
if n == limit {
|
|
break 'files;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// `at` is the local time with its offset in one format, so the text order is the time order.
|
|
out.sort_by(|a, b| b["at"].as_str().unwrap_or("").cmp(a["at"].as_str().unwrap_or("")));
|
|
out.truncate(limit);
|
|
Ok(out)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::datadir::temp_dir;
|
|
use crate::settings::parse_settings;
|
|
|
|
fn settings(v: Value) -> Settings {
|
|
parse_settings(&v).unwrap()
|
|
}
|
|
|
|
#[test]
|
|
fn the_worker_gets_only_enabled_targets_and_the_keep_days_also_without_targets() {
|
|
let data = temp_dir("collect-cfg");
|
|
let s = settings(json!({"collect": {"keepDays": 3, "targets": [
|
|
{"name": "a", "kind": "rss", "value": "https://a.example/rss", "everyMinutes": 30},
|
|
{"name": "b", "kind": "site", "value": "https://b.example", "enabled": false}]}}));
|
|
let read = |f: &Path| serde_json::from_str::<Value>(&std::fs::read_to_string(f).unwrap()).unwrap();
|
|
let dir = data.join("collect").to_string_lossy().into_owned();
|
|
let file = write_config(&data, &s).unwrap();
|
|
assert_eq!(read(&file), json!({"dir": dir, "keepDays": 3,
|
|
"targets": [{"name": "a", "kind": "rss", "value": "https://a.example/rss", "everyMinutes": 30}]}));
|
|
// Feature off or no targets: nothing to collect, but old results still go after keepDays.
|
|
let mut off = s.clone();
|
|
off.features.collect = false;
|
|
assert_eq!(read(&write_config(&data, &off).unwrap()), json!({"dir": dir, "keepDays": 3, "targets": []}));
|
|
assert_eq!(read(&write_config(&data, &settings(json!({}))).unwrap()), json!({"dir": dir, "keepDays": 30, "targets": []}));
|
|
}
|
|
|
|
#[test]
|
|
fn results_come_newest_first_with_their_target_and_only_from_target_folders() {
|
|
let data = temp_dir("collect-res");
|
|
let put = |name: &str, day: &str, lines: &[&str]| {
|
|
let d = data.join("collect").join(name);
|
|
std::fs::create_dir_all(&d).unwrap();
|
|
std::fs::write(d.join(format!("{day}.jsonl")), lines.join("\n") + "\n").unwrap();
|
|
};
|
|
put("뉴스", "2026-11-12", &[r#"{"at": "2026-11-12T09:00:00+09:00", "title": "old"}"#]);
|
|
put("뉴스", "2026-11-13", &[r#"{"at": "2026-11-13T09:00:00+09:00", "title": "n1"}"#, "not json",
|
|
r#"{"at": "2026-11-13T10:00:00+09:00", "title": "n2"}"#]);
|
|
put("site", "2026-11-13", &[r#"{"at": "2026-11-13T09:30:00+09:00", "title": "s1"}"#]);
|
|
std::fs::write(data.join("collect").join("뉴스").join("state.json"), r#"{"at": "9999"}"#).unwrap();
|
|
let titles = |v: Vec<Value>| v.iter().map(|i| format!("{}:{}", i["target"].as_str().unwrap(),
|
|
i["title"].as_str().unwrap())).collect::<Vec<_>>();
|
|
assert_eq!(titles(results(&data, None, 100).unwrap()), ["뉴스:n2", "site:s1", "뉴스:n1", "뉴스:old"]);
|
|
assert_eq!(titles(results(&data, Some("뉴스"), 2).unwrap()), ["뉴스:n2", "뉴스:n1"]);
|
|
assert_eq!(titles(results(&data, None, 2).unwrap()), ["뉴스:n2", "site:s1"]);
|
|
assert!(results(&data, Some("missing"), 10).unwrap().is_empty());
|
|
// Every target with results is listed, also one whose items are all older than the newest.
|
|
std::fs::create_dir_all(data.join("collect").join("empty")).unwrap();
|
|
assert_eq!(result_targets(&data), ["site", "뉴스"]);
|
|
assert!(results(&data, Some(".."), 10).is_err());
|
|
assert!(results(&data, Some("../config"), 10).is_err());
|
|
}
|
|
}
|