feat(app): background collection frame for sites, search words and RSS feeds

Targets (none by default), interval and keep days on a new 백그라운드 수집 screen with the
results; the worker collects on its own thread through the web search's address check and
writes only to the data folder's collect/ (logs carry target names and counts).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-13 21:29:49 +09:00
parent d531eb0ffc
commit b94e9dc112
15 changed files with 861 additions and 14 deletions

View File

@@ -67,6 +67,13 @@ def main() -> int:
brain = JarvisBrain(language=args.tool_language or None, reply_language=args.reply_language or None,
mcp_env=mcp_env)
# Background collection runs apart from the voice and the brain (voice_worker.spec.md).
collector = None
if os.environ.get("JARVIS_COLLECT_PATH"):
from .collect import Collector, load_config
collector = Collector(load_config(os.environ["JARVIS_COLLECT_PATH"]))
collector.start()
async def run() -> None:
worker = VoiceWorker(token, transcriber, vad.stream, tts, history=history, brain=brain)
server = await worker.serve(args.host, args.port)
@@ -81,6 +88,8 @@ def main() -> int:
except KeyboardInterrupt:
pass
finally:
if collector:
collector.stop()
if tts:
tts.close()
if history:

241
src/voice_worker/collect.py Normal file
View File

@@ -0,0 +1,241 @@
"""Background collection of sites, search words and RSS feeds (voice_worker.spec.md, "Background
collection"). Results go only to day files in the data folder; logs carry target names and counts."""
from __future__ import annotations
import hashlib
import json
import re
import threading
import time
import urllib.parse
import xml.etree.ElementTree as ET
from dataclasses import dataclass
from datetime import date, datetime, timedelta
from pathlib import Path
from typing import Callable, Optional
from .log import debug_log, notice
MAX_BYTES = 1024 * 1024
TIMEOUT_S = 15
MAX_REDIRECTS = 5
SITE_CHARS = 20_000
SUMMARY_CHARS = 2_000
SEEN_KEEP = 1000
USER_AGENT = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36")
_DAY_FILE = re.compile(r"^(\d{4}-\d{2}-\d{2})\.jsonl$")
@dataclass
class Target:
name: str
kind: str
value: str
every_s: float
@dataclass
class Config:
dir: Path
keep_days: int
targets: list
def load_config(path) -> Config:
raw = json.loads(Path(path).read_text(encoding="utf-8"))
return Config(Path(raw["dir"]), int(raw.get("keepDays", 30)),
[Target(t["name"], t["kind"], t["value"], float(t["everyMinutes"]) * 60) for t in raw.get("targets", [])])
def safe_get(url: str) -> bytes:
"""GET a public http(s) address: every redirect checked again, at most ``MAX_BYTES``."""
import requests
from jarvis.tools.builtin.web_search import _is_public_url
for _ in range(MAX_REDIRECTS + 1):
if not _is_public_url(url):
raise ValueError("not a public address")
with requests.get(url, headers={"User-Agent": USER_AGENT}, timeout=TIMEOUT_S,
allow_redirects=False, stream=True) as r:
if r.is_redirect or r.is_permanent_redirect:
url = urllib.parse.urljoin(url, r.headers.get("Location", ""))
continue
r.raise_for_status()
if r.status_code != 200: # e.g. DuckDuckGo's 202 challenge
raise ValueError(f"status {r.status_code}")
body = b""
for chunk in r.iter_content(64 * 1024):
body += chunk
if len(body) > MAX_BYTES:
raise ValueError("response too large")
return body
raise ValueError("too many redirects")
def _text(html) -> str:
from bs4 import BeautifulSoup
soup = BeautifulSoup(html, "html.parser")
for tag in soup(["script", "style", "noscript"]):
tag.decompose()
return " ".join(soup.get_text(" ").split())
def _site(body: bytes) -> list:
from bs4 import BeautifulSoup
soup = BeautifulSoup(body, "html.parser")
title = soup.title.get_text().strip() if soup.title else ""
for tag in soup(["script", "style", "noscript", "title"]):
tag.decompose()
text = "\n".join(line for line in (" ".join(l.split()) for l in soup.get_text("\n").splitlines()) if line)
return [{"title": title, "text": text[:SITE_CHARS]}]
def _local(tag: str) -> str:
return tag.rsplit("}", 1)[-1]
def _feed(body: bytes) -> list:
"""RSS 2.0 items or Atom entries, each with an ``id`` to tell new ones."""
out = []
for el in ET.fromstring(body).iter():
if _local(el.tag) not in ("item", "entry"):
continue
f = {_local(c.tag): c for c in el}
link = (f["link"].get("href") or (f["link"].text or "")).strip() if "link" in f else ""
get = lambda *keys: next(((f[k].text or "").strip() for k in keys if k in f), "")
item = {"title": get("title"), "url": link, "published": get("pubDate", "published", "updated"),
"text": _text(get("description", "summary", "content"))[:SUMMARY_CHARS]}
item["id"] = get("guid", "id") or link or item["title"]
out.append(item)
return out
def search_url(words: str) -> str:
return "https://lite.duckduckgo.com/lite/?q=" + urllib.parse.quote_plus(words)
class SearchBlocked(Exception):
"""DuckDuckGo served its bot challenge instead of results (web_search.spec.md)."""
def _search(body: bytes) -> list:
from bs4 import BeautifulSoup
if b"anomaly-modal" in body or b"anomaly.js" in body:
raise SearchBlocked()
out = []
for a in BeautifulSoup(body, "html.parser").find_all("a", href=True):
href, title = a["href"], " ".join(a.get_text().split())
if href.startswith("//duckduckgo.com/l/"):
href = urllib.parse.parse_qs(urllib.parse.urlparse(href).query).get("uddg", [""])[0]
host = urllib.parse.urlparse(href).hostname or ""
if href.startswith("http") and len(title) > 10 and not host.endswith("duckduckgo.com"):
out.append({"title": title, "url": href, "id": href})
return out[:10]
class Collector:
def __init__(self, cfg: Config, fetch: Callable[[str], bytes] = safe_get, check_every: float = 30.0):
self.cfg = cfg
self.fetch = fetch
self.check_every = check_every
self._stop = threading.Event()
self._thread: Optional[threading.Thread] = None
# -- state ----------------------------------------------------------------------------------
def _folder(self, t: Target) -> Path:
return self.cfg.dir / t.name
def _state(self, t: Target) -> dict:
try:
return json.loads((self._folder(t) / "state.json").read_text(encoding="utf-8"))
except (OSError, ValueError):
return {}
def _save_state(self, t: Target, state: dict) -> None:
f = self._folder(t) / "state.json"
f.parent.mkdir(parents=True, exist_ok=True)
tmp = f.with_suffix(".tmp")
tmp.write_text(json.dumps(state, ensure_ascii=False), encoding="utf-8")
tmp.replace(f)
def _keep(self, t: Target, new: list) -> None:
if not new:
return
f = self._folder(t) / f"{date.today()}.jsonl"
f.parent.mkdir(parents=True, exist_ok=True)
at = datetime.now().astimezone().isoformat(timespec="seconds")
with f.open("a", encoding="utf-8") as out:
for item in new:
out.write(json.dumps({"at": at, **item}, ensure_ascii=False) + "\n")
# -- one round ------------------------------------------------------------------------------
def _prune(self) -> None:
oldest = date.today() - timedelta(days=self.cfg.keep_days)
if not self.cfg.dir.is_dir():
return
for folder in self.cfg.dir.iterdir():
if not folder.is_dir():
continue
for f in folder.iterdir():
m = _DAY_FILE.match(f.name)
if m and date.fromisoformat(m.group(1)) < oldest:
f.unlink(missing_ok=True)
def _run(self, t: Target, state: dict) -> int:
if t.kind == "site":
[page] = _site(self.fetch(t.value))
digest = hashlib.sha256(page["text"].encode()).hexdigest()
if digest == state.get("hash"):
return 0
state["hash"] = digest
self._keep(t, [{**page, "url": t.value}])
return 1
found = _feed(self.fetch(t.value)) if t.kind == "rss" else _search(self.fetch(search_url(t.value)))
seen = state.get("seen", [])
new = [i for i in found if i["id"] not in seen]
state["seen"] = (seen + [i["id"] for i in new])[-SEEN_KEEP:]
self._keep(t, [{k: v for k, v in i.items() if k != "id"} for i in new])
return len(new)
def run_due(self, now: Optional[float] = None) -> None:
now = time.time() if now is None else now
self._prune()
for t in self.cfg.targets:
if self._stop.is_set():
return
state = self._state(t)
if now - state.get("last", float("-inf")) < t.every_s:
continue
state["last"] = now
t0 = time.perf_counter()
try:
n = self._run(t, state)
notice(f"collect {t.name}: {n} new", "voice_worker")
except Exception as e: # noqa: BLE001 - a broken target waits for its next interval
notice(f"Warning: collect {t.name} failed: {type(e).__name__}", "voice_worker")
self._save_state(t, state)
debug_log(f"collect {t.name} kind={t.kind} ms={(time.perf_counter() - t0) * 1000:.0f}", "voice_worker")
# -- thread ---------------------------------------------------------------------------------
def start(self) -> None:
def loop():
while not self._stop.is_set():
try:
self.run_due()
except Exception as e: # noqa: BLE001 - e.g. the folder became unwritable
notice(f"Warning: collect round failed: {type(e).__name__}", "voice_worker")
self._stop.wait(self.check_every)
self._thread = threading.Thread(target=loop, name="collect", daemon=True)
self._thread.start()
debug_log(f"collect started: {len(self.cfg.targets)} target(s)", "voice_worker")
def stop(self, timeout: float = 5.0) -> None:
self._stop.set()
if self._thread:
self._thread.join(timeout)
def running(self) -> bool:
return bool(self._thread and self._thread.is_alive())

View File

@@ -317,6 +317,29 @@ which carry Discord's own time.
`budget_tokens` (4000).
- Logs carry IDs, counts and timings only, never record, question or answer text.
### Background collection (M6)
With `JARVIS_COLLECT_PATH` set, the worker collects on its own thread, apart from the voice and the
brain (`app/src/admin.spec.md`, "Background collection"). The file is
`{"dir": "<data>/collect", "keepDays": 30, "targets": [{"name", "kind", "value", "everyMinutes"}]}`;
the app writes only enabled targets. Nothing is collected without targets.
- A target runs when `everyMinutes` passed since its last run (also across restarts; the last run is
kept in `<dir>/<name>/state.json`), one target at a time, checked every 30 s. A failed run counts
as a run, so a broken site is tried again only at its next interval.
- Kinds: `site` keeps the page's title and text (at most 20,000 characters) when it differs from the
last kept one; `rss` (RSS or Atom) keeps each new item (title, link, time, summary of at most 2,000
characters, new by its id or link); `search` keeps each new web search result (title and link) for
the words, from DuckDuckGo's lite page, which needs no key (its bot challenge is a failed run, not "no results").
- Every request goes through the web search's address check (`web_search.spec.md`): only public
http(s) addresses, every redirect checked again, at most 1 MB, 15 s. The collected text is web
content: it goes only to `<dir>/<name>/<YYYY-MM-DD>.jsonl` (one JSON object per line: `at`, then
`title`, `url`, `text` as the kind has them), never to the brain or a log.
- Day files older than `keepDays` are deleted in every folder of `dir` at each check, also of
targets no longer listed.
- Each run is a `notice` line (category `voice_worker`): the target's name and how many new items, or
a `Warning` with the error's type. Never a URL, words or collected text.
### Errors
`{"type": "error", "code": "...", "message": "...", "id": "..."}`. Codes: `bad_message` (unknown type,