feat(app): background collection frame for sites, search words and RSS feeds
Targets (none by default), interval and keep days on a new 백그라운드 수집 screen with the results; the worker collects on its own thread through the web search's address check and writes only to the data folder's collect/ (logs carry target names and counts). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -67,6 +67,13 @@ def main() -> int:
|
||||
brain = JarvisBrain(language=args.tool_language or None, reply_language=args.reply_language or None,
|
||||
mcp_env=mcp_env)
|
||||
|
||||
# Background collection runs apart from the voice and the brain (voice_worker.spec.md).
|
||||
collector = None
|
||||
if os.environ.get("JARVIS_COLLECT_PATH"):
|
||||
from .collect import Collector, load_config
|
||||
collector = Collector(load_config(os.environ["JARVIS_COLLECT_PATH"]))
|
||||
collector.start()
|
||||
|
||||
async def run() -> None:
|
||||
worker = VoiceWorker(token, transcriber, vad.stream, tts, history=history, brain=brain)
|
||||
server = await worker.serve(args.host, args.port)
|
||||
@@ -81,6 +88,8 @@ def main() -> int:
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
if collector:
|
||||
collector.stop()
|
||||
if tts:
|
||||
tts.close()
|
||||
if history:
|
||||
|
||||
241
src/voice_worker/collect.py
Normal file
241
src/voice_worker/collect.py
Normal file
@@ -0,0 +1,241 @@
|
||||
"""Background collection of sites, search words and RSS feeds (voice_worker.spec.md, "Background
|
||||
collection"). Results go only to day files in the data folder; logs carry target names and counts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
import xml.etree.ElementTree as ET
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional
|
||||
|
||||
from .log import debug_log, notice
|
||||
|
||||
MAX_BYTES = 1024 * 1024
|
||||
TIMEOUT_S = 15
|
||||
MAX_REDIRECTS = 5
|
||||
SITE_CHARS = 20_000
|
||||
SUMMARY_CHARS = 2_000
|
||||
SEEN_KEEP = 1000
|
||||
USER_AGENT = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36")
|
||||
_DAY_FILE = re.compile(r"^(\d{4}-\d{2}-\d{2})\.jsonl$")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Target:
|
||||
name: str
|
||||
kind: str
|
||||
value: str
|
||||
every_s: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
dir: Path
|
||||
keep_days: int
|
||||
targets: list
|
||||
|
||||
|
||||
def load_config(path) -> Config:
|
||||
raw = json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
return Config(Path(raw["dir"]), int(raw.get("keepDays", 30)),
|
||||
[Target(t["name"], t["kind"], t["value"], float(t["everyMinutes"]) * 60) for t in raw.get("targets", [])])
|
||||
|
||||
|
||||
def safe_get(url: str) -> bytes:
|
||||
"""GET a public http(s) address: every redirect checked again, at most ``MAX_BYTES``."""
|
||||
import requests
|
||||
from jarvis.tools.builtin.web_search import _is_public_url
|
||||
|
||||
for _ in range(MAX_REDIRECTS + 1):
|
||||
if not _is_public_url(url):
|
||||
raise ValueError("not a public address")
|
||||
with requests.get(url, headers={"User-Agent": USER_AGENT}, timeout=TIMEOUT_S,
|
||||
allow_redirects=False, stream=True) as r:
|
||||
if r.is_redirect or r.is_permanent_redirect:
|
||||
url = urllib.parse.urljoin(url, r.headers.get("Location", ""))
|
||||
continue
|
||||
r.raise_for_status()
|
||||
if r.status_code != 200: # e.g. DuckDuckGo's 202 challenge
|
||||
raise ValueError(f"status {r.status_code}")
|
||||
body = b""
|
||||
for chunk in r.iter_content(64 * 1024):
|
||||
body += chunk
|
||||
if len(body) > MAX_BYTES:
|
||||
raise ValueError("response too large")
|
||||
return body
|
||||
raise ValueError("too many redirects")
|
||||
|
||||
|
||||
def _text(html) -> str:
|
||||
from bs4 import BeautifulSoup
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
for tag in soup(["script", "style", "noscript"]):
|
||||
tag.decompose()
|
||||
return " ".join(soup.get_text(" ").split())
|
||||
|
||||
|
||||
def _site(body: bytes) -> list:
|
||||
from bs4 import BeautifulSoup
|
||||
soup = BeautifulSoup(body, "html.parser")
|
||||
title = soup.title.get_text().strip() if soup.title else ""
|
||||
for tag in soup(["script", "style", "noscript", "title"]):
|
||||
tag.decompose()
|
||||
text = "\n".join(line for line in (" ".join(l.split()) for l in soup.get_text("\n").splitlines()) if line)
|
||||
return [{"title": title, "text": text[:SITE_CHARS]}]
|
||||
|
||||
|
||||
def _local(tag: str) -> str:
|
||||
return tag.rsplit("}", 1)[-1]
|
||||
|
||||
|
||||
def _feed(body: bytes) -> list:
|
||||
"""RSS 2.0 items or Atom entries, each with an ``id`` to tell new ones."""
|
||||
out = []
|
||||
for el in ET.fromstring(body).iter():
|
||||
if _local(el.tag) not in ("item", "entry"):
|
||||
continue
|
||||
f = {_local(c.tag): c for c in el}
|
||||
link = (f["link"].get("href") or (f["link"].text or "")).strip() if "link" in f else ""
|
||||
get = lambda *keys: next(((f[k].text or "").strip() for k in keys if k in f), "")
|
||||
item = {"title": get("title"), "url": link, "published": get("pubDate", "published", "updated"),
|
||||
"text": _text(get("description", "summary", "content"))[:SUMMARY_CHARS]}
|
||||
item["id"] = get("guid", "id") or link or item["title"]
|
||||
out.append(item)
|
||||
return out
|
||||
|
||||
|
||||
def search_url(words: str) -> str:
|
||||
return "https://lite.duckduckgo.com/lite/?q=" + urllib.parse.quote_plus(words)
|
||||
|
||||
|
||||
class SearchBlocked(Exception):
|
||||
"""DuckDuckGo served its bot challenge instead of results (web_search.spec.md)."""
|
||||
|
||||
|
||||
def _search(body: bytes) -> list:
|
||||
from bs4 import BeautifulSoup
|
||||
if b"anomaly-modal" in body or b"anomaly.js" in body:
|
||||
raise SearchBlocked()
|
||||
out = []
|
||||
for a in BeautifulSoup(body, "html.parser").find_all("a", href=True):
|
||||
href, title = a["href"], " ".join(a.get_text().split())
|
||||
if href.startswith("//duckduckgo.com/l/"):
|
||||
href = urllib.parse.parse_qs(urllib.parse.urlparse(href).query).get("uddg", [""])[0]
|
||||
host = urllib.parse.urlparse(href).hostname or ""
|
||||
if href.startswith("http") and len(title) > 10 and not host.endswith("duckduckgo.com"):
|
||||
out.append({"title": title, "url": href, "id": href})
|
||||
return out[:10]
|
||||
|
||||
|
||||
class Collector:
|
||||
def __init__(self, cfg: Config, fetch: Callable[[str], bytes] = safe_get, check_every: float = 30.0):
|
||||
self.cfg = cfg
|
||||
self.fetch = fetch
|
||||
self.check_every = check_every
|
||||
self._stop = threading.Event()
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
|
||||
# -- state ----------------------------------------------------------------------------------
|
||||
def _folder(self, t: Target) -> Path:
|
||||
return self.cfg.dir / t.name
|
||||
|
||||
def _state(self, t: Target) -> dict:
|
||||
try:
|
||||
return json.loads((self._folder(t) / "state.json").read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
|
||||
def _save_state(self, t: Target, state: dict) -> None:
|
||||
f = self._folder(t) / "state.json"
|
||||
f.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = f.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(state, ensure_ascii=False), encoding="utf-8")
|
||||
tmp.replace(f)
|
||||
|
||||
def _keep(self, t: Target, new: list) -> None:
|
||||
if not new:
|
||||
return
|
||||
f = self._folder(t) / f"{date.today()}.jsonl"
|
||||
f.parent.mkdir(parents=True, exist_ok=True)
|
||||
at = datetime.now().astimezone().isoformat(timespec="seconds")
|
||||
with f.open("a", encoding="utf-8") as out:
|
||||
for item in new:
|
||||
out.write(json.dumps({"at": at, **item}, ensure_ascii=False) + "\n")
|
||||
|
||||
# -- one round ------------------------------------------------------------------------------
|
||||
def _prune(self) -> None:
|
||||
oldest = date.today() - timedelta(days=self.cfg.keep_days)
|
||||
if not self.cfg.dir.is_dir():
|
||||
return
|
||||
for folder in self.cfg.dir.iterdir():
|
||||
if not folder.is_dir():
|
||||
continue
|
||||
for f in folder.iterdir():
|
||||
m = _DAY_FILE.match(f.name)
|
||||
if m and date.fromisoformat(m.group(1)) < oldest:
|
||||
f.unlink(missing_ok=True)
|
||||
|
||||
def _run(self, t: Target, state: dict) -> int:
|
||||
if t.kind == "site":
|
||||
[page] = _site(self.fetch(t.value))
|
||||
digest = hashlib.sha256(page["text"].encode()).hexdigest()
|
||||
if digest == state.get("hash"):
|
||||
return 0
|
||||
state["hash"] = digest
|
||||
self._keep(t, [{**page, "url": t.value}])
|
||||
return 1
|
||||
found = _feed(self.fetch(t.value)) if t.kind == "rss" else _search(self.fetch(search_url(t.value)))
|
||||
seen = state.get("seen", [])
|
||||
new = [i for i in found if i["id"] not in seen]
|
||||
state["seen"] = (seen + [i["id"] for i in new])[-SEEN_KEEP:]
|
||||
self._keep(t, [{k: v for k, v in i.items() if k != "id"} for i in new])
|
||||
return len(new)
|
||||
|
||||
def run_due(self, now: Optional[float] = None) -> None:
|
||||
now = time.time() if now is None else now
|
||||
self._prune()
|
||||
for t in self.cfg.targets:
|
||||
if self._stop.is_set():
|
||||
return
|
||||
state = self._state(t)
|
||||
if now - state.get("last", float("-inf")) < t.every_s:
|
||||
continue
|
||||
state["last"] = now
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
n = self._run(t, state)
|
||||
notice(f"collect {t.name}: {n} new", "voice_worker")
|
||||
except Exception as e: # noqa: BLE001 - a broken target waits for its next interval
|
||||
notice(f"Warning: collect {t.name} failed: {type(e).__name__}", "voice_worker")
|
||||
self._save_state(t, state)
|
||||
debug_log(f"collect {t.name} kind={t.kind} ms={(time.perf_counter() - t0) * 1000:.0f}", "voice_worker")
|
||||
|
||||
# -- thread ---------------------------------------------------------------------------------
|
||||
def start(self) -> None:
|
||||
def loop():
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
self.run_due()
|
||||
except Exception as e: # noqa: BLE001 - e.g. the folder became unwritable
|
||||
notice(f"Warning: collect round failed: {type(e).__name__}", "voice_worker")
|
||||
self._stop.wait(self.check_every)
|
||||
|
||||
self._thread = threading.Thread(target=loop, name="collect", daemon=True)
|
||||
self._thread.start()
|
||||
debug_log(f"collect started: {len(self.cfg.targets)} target(s)", "voice_worker")
|
||||
|
||||
def stop(self, timeout: float = 5.0) -> None:
|
||||
self._stop.set()
|
||||
if self._thread:
|
||||
self._thread.join(timeout)
|
||||
|
||||
def running(self) -> bool:
|
||||
return bool(self._thread and self._thread.is_alive())
|
||||
@@ -317,6 +317,29 @@ which carry Discord's own time.
|
||||
`budget_tokens` (4000).
|
||||
- Logs carry IDs, counts and timings only, never record, question or answer text.
|
||||
|
||||
### Background collection (M6)
|
||||
|
||||
With `JARVIS_COLLECT_PATH` set, the worker collects on its own thread, apart from the voice and the
|
||||
brain (`app/src/admin.spec.md`, "Background collection"). The file is
|
||||
`{"dir": "<data>/collect", "keepDays": 30, "targets": [{"name", "kind", "value", "everyMinutes"}]}`;
|
||||
the app writes only enabled targets. Nothing is collected without targets.
|
||||
|
||||
- A target runs when `everyMinutes` passed since its last run (also across restarts; the last run is
|
||||
kept in `<dir>/<name>/state.json`), one target at a time, checked every 30 s. A failed run counts
|
||||
as a run, so a broken site is tried again only at its next interval.
|
||||
- Kinds: `site` keeps the page's title and text (at most 20,000 characters) when it differs from the
|
||||
last kept one; `rss` (RSS or Atom) keeps each new item (title, link, time, summary of at most 2,000
|
||||
characters, new by its id or link); `search` keeps each new web search result (title and link) for
|
||||
the words, from DuckDuckGo's lite page, which needs no key (its bot challenge is a failed run, not "no results").
|
||||
- Every request goes through the web search's address check (`web_search.spec.md`): only public
|
||||
http(s) addresses, every redirect checked again, at most 1 MB, 15 s. The collected text is web
|
||||
content: it goes only to `<dir>/<name>/<YYYY-MM-DD>.jsonl` (one JSON object per line: `at`, then
|
||||
`title`, `url`, `text` as the kind has them), never to the brain or a log.
|
||||
- Day files older than `keepDays` are deleted in every folder of `dir` at each check, also of
|
||||
targets no longer listed.
|
||||
- Each run is a `notice` line (category `voice_worker`): the target's name and how many new items, or
|
||||
a `Warning` with the error's type. Never a URL, words or collected text.
|
||||
|
||||
### Errors
|
||||
|
||||
`{"type": "error", "code": "...", "message": "...", "id": "..."}`. Codes: `bad_message` (unknown type,
|
||||
|
||||
Reference in New Issue
Block a user