//! Background collection (`app/src/admin.spec.md`, "Background collection"): the worker's //! collection file, and the collected items for the 백그라운드 수집 screen. use std::path::{Path, PathBuf}; use serde_json::{json, Value}; use crate::settings::{valid_collect_name, Settings}; /// The worker's environment variable with the collection file (`voice_worker/collect.py`). pub const COLLECT_ENV_VAR: &str = "JARVIS_COLLECT_PATH"; /// Write `config/collect.json` and return it. With the feature off it lists no targets: the worker /// then collects nothing but still deletes day files older than the keep days. pub fn write_config(data: &Path, settings: &Settings) -> std::io::Result { let file = data.join("config").join("collect.json"); let targets: Vec = settings.collect.targets.iter().filter(|t| t.enabled && settings.features.collect) .map(|t| json!({"name": t.name, "kind": t.kind, "value": t.value, "everyMinutes": t.every_minutes})) .collect(); let cfg = json!({"dir": data.join("collect").to_string_lossy(), "keepDays": settings.collect.keep_days, "targets": targets}); let mut text = serde_json::to_string_pretty(&cfg).unwrap(); text.push('\n'); crate::datadir::write_atomic(&file, text.as_bytes())?; Ok(file) } /// The day files of one target folder, newest first. fn day_files(folder: &Path) -> Vec { let mut files: Vec = std::fs::read_dir(folder).into_iter().flatten().flatten().map(|e| e.path()) .filter(|p| p.file_name().and_then(|n| n.to_str()).is_some_and(|n| n.len() == 16 && n.ends_with(".jsonl"))) .collect(); files.sort(); files.reverse(); files } /// The target folders that hold results, sorted. pub fn result_targets(data: &Path) -> Vec { let root = data.join("collect"); let mut names: Vec = std::fs::read_dir(&root).into_iter().flatten().flatten() .filter(|e| e.path().is_dir()) .filter_map(|e| e.file_name().to_str().map(str::to_owned)) .filter(|n| valid_collect_name(n) && !day_files(&root.join(n)).is_empty()) .collect(); names.sort(); names } /// Collected items newest first, each with its `target`; `target` None reads every target folder. pub fn results(data: &Path, target: Option<&str>, limit: usize) -> Result, String> { let root = data.join("collect"); let names: Vec = match target { Some(t) if !valid_collect_name(t) => return Err(format!("not a collection target name: {t}")), Some(t) => vec![t.to_owned()], None => result_targets(data), }; let mut out: Vec = Vec::new(); for name in names { let mut n = 0; 'files: for f in day_files(&root.join(&name)) { let text = std::fs::read_to_string(&f).unwrap_or_default(); for line in text.lines().rev() { if let Ok(Value::Object(mut item)) = serde_json::from_str::(line) { item.insert("target".into(), Value::String(name.clone())); out.push(Value::Object(item)); n += 1; if n == limit { break 'files; } } } } } // `at` is the local time with its offset in one format, so the text order is the time order. out.sort_by(|a, b| b["at"].as_str().unwrap_or("").cmp(a["at"].as_str().unwrap_or(""))); out.truncate(limit); Ok(out) } #[cfg(test)] mod tests { use super::*; use crate::datadir::temp_dir; use crate::settings::parse_settings; fn settings(v: Value) -> Settings { parse_settings(&v).unwrap() } #[test] fn the_worker_gets_only_enabled_targets_and_the_keep_days_also_without_targets() { let data = temp_dir("collect-cfg"); let s = settings(json!({"collect": {"keepDays": 3, "targets": [ {"name": "a", "kind": "rss", "value": "https://a.example/rss", "everyMinutes": 30}, {"name": "b", "kind": "site", "value": "https://b.example", "enabled": false}]}})); let read = |f: &Path| serde_json::from_str::(&std::fs::read_to_string(f).unwrap()).unwrap(); let dir = data.join("collect").to_string_lossy().into_owned(); let file = write_config(&data, &s).unwrap(); assert_eq!(read(&file), json!({"dir": dir, "keepDays": 3, "targets": [{"name": "a", "kind": "rss", "value": "https://a.example/rss", "everyMinutes": 30}]})); // Feature off or no targets: nothing to collect, but old results still go after keepDays. let mut off = s.clone(); off.features.collect = false; assert_eq!(read(&write_config(&data, &off).unwrap()), json!({"dir": dir, "keepDays": 3, "targets": []})); assert_eq!(read(&write_config(&data, &settings(json!({}))).unwrap()), json!({"dir": dir, "keepDays": 30, "targets": []})); } #[test] fn results_come_newest_first_with_their_target_and_only_from_target_folders() { let data = temp_dir("collect-res"); let put = |name: &str, day: &str, lines: &[&str]| { let d = data.join("collect").join(name); std::fs::create_dir_all(&d).unwrap(); std::fs::write(d.join(format!("{day}.jsonl")), lines.join("\n") + "\n").unwrap(); }; put("뉴스", "2026-11-12", &[r#"{"at": "2026-11-12T09:00:00+09:00", "title": "old"}"#]); put("뉴스", "2026-11-13", &[r#"{"at": "2026-11-13T09:00:00+09:00", "title": "n1"}"#, "not json", r#"{"at": "2026-11-13T10:00:00+09:00", "title": "n2"}"#]); put("site", "2026-11-13", &[r#"{"at": "2026-11-13T09:30:00+09:00", "title": "s1"}"#]); std::fs::write(data.join("collect").join("뉴스").join("state.json"), r#"{"at": "9999"}"#).unwrap(); let titles = |v: Vec| v.iter().map(|i| format!("{}:{}", i["target"].as_str().unwrap(), i["title"].as_str().unwrap())).collect::>(); assert_eq!(titles(results(&data, None, 100).unwrap()), ["뉴스:n2", "site:s1", "뉴스:n1", "뉴스:old"]); assert_eq!(titles(results(&data, Some("뉴스"), 2).unwrap()), ["뉴스:n2", "뉴스:n1"]); assert_eq!(titles(results(&data, None, 2).unwrap()), ["뉴스:n2", "site:s1"]); assert!(results(&data, Some("missing"), 10).unwrap().is_empty()); // Every target with results is listed, also one whose items are all older than the newest. std::fs::create_dir_all(data.join("collect").join("empty")).unwrap(); assert_eq!(result_targets(&data), ["site", "뉴스"]); assert!(results(&data, Some(".."), 10).is_err()); assert!(results(&data, Some("../config"), 10).is_err()); } }