fix(app): log hub masks dotted tokens, keeps worker errors and drops the oldest lines by time

- redact: a token with sentence dots around it is masked too.
- Worker error messages are always logged (code and id), so the log screen
  and the error report show them without debug logging.
- The voice worker logs with stt, llm and tts categories, so its lines land
  in those tabs.
- The zip leaves out the oldest lines by the time they were logged, across
  every tab's file, not by file name.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-12 13:43:01 +09:00
parent 390f6f0b86
commit eb0fa2f159
13 changed files with 143 additions and 37 deletions

View File

@@ -81,9 +81,12 @@ STT, LLM and TTS lines go to `stt`, `llm`, `tts` by their worker category), `bro
order), the app version and a short PC summary (OS, CPU count, memory); `None` without an error.
When it is over 2,000 characters (Discord's message limit) the screen offers a download instead.
- `export_zip(from, to, out)`: the log files of those days in a zip, at most 10 MB (Discord's file
limit): when larger, the oldest lines are left out and a note says so.
limit): when larger, the oldest lines by the time they were logged, across every tab's file, are
left out and a note says how many.
- Every copy and export runs through `redact`: each stored secret's value, and anything shaped like
a Discord bot token, becomes `[비밀값]`.
a Discord bot token (also with sentence dots around it), becomes `[비밀값]`.
- Errors the voice worker reports to the app (`error` messages: code and request id) are always
logged, debug logging or not, so the log screen and the error report show them.
## Tests

View File

@@ -213,19 +213,25 @@ impl LogHub {
}
}
/// The zip with the oldest `dropped` lines (oldest files first) left out.
/// The zip with the `dropped` oldest lines left out: oldest by the time each line was logged, across
/// every tab's file (a line's time is its first 23 characters, `YYYY-MM-DD HH:MM:SS.mmm`; lines
/// without one count as oldest).
fn build_zip(files: &[(NaiveDate, String, Vec<String>)], dropped: usize) -> std::io::Result<Vec<u8>> {
let mut order: Vec<(&str, usize, usize)> = files.iter().enumerate()
.flat_map(|(fi, (_, _, lines))| lines.iter().enumerate()
.map(move |(li, l)| (l.get(..23).filter(|t| line_time_ok(t)).unwrap_or(""), fi, li)))
.collect();
order.sort();
let gone: std::collections::HashSet<(usize, usize)> = order.iter().take(dropped).map(|&(_, f, l)| (f, l)).collect();
let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
let opts = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated);
let mut skip = dropped;
for (_, name, lines) in files {
let cut = skip.min(lines.len());
skip -= cut;
if cut == lines.len() {
for (fi, (_, name, lines)) in files.iter().enumerate() {
let kept: Vec<&String> = lines.iter().enumerate().filter(|(li, _)| !gone.contains(&(fi, *li))).map(|(_, l)| l).collect();
if kept.is_empty() {
continue;
}
zip.start_file(name.as_str(), opts).map_err(std::io::Error::other)?;
for l in &lines[cut..] {
for l in kept {
zip.write_all(l.as_bytes())?;
zip.write_all(b"\n")?;
}
@@ -237,6 +243,10 @@ fn build_zip(files: &[(NaiveDate, String, Vec<String>)], dropped: usize) -> std:
Ok(zip.finish().map_err(std::io::Error::other)?.into_inner())
}
fn line_time_ok(t: &str) -> bool {
chrono::NaiveDateTime::parse_from_str(t, "%Y-%m-%d %H:%M:%S%.3f").is_ok()
}
fn file_day(name: &str) -> Option<NaiveDate> {
let stem = name.strip_suffix(".log")?;
let date = stem.get(stem.len().checked_sub(10)?..)?;
@@ -269,8 +279,18 @@ pub fn redact(text: &str, secrets: &[String]) -> String {
}
let mut result = String::with_capacity(out.len());
let mut word = String::new();
// A word may carry sentence dots around it ("token=<token>."): they are not part of the token.
let flush = |word: &mut String, result: &mut String| {
if looks_like_token(word) { result.push_str(SECRET_MARK) } else { result.push_str(word) }
let core = word.trim_matches('.');
if looks_like_token(core) {
let lead = word.len() - word.trim_start_matches('.').len();
let trail = word.len() - word.trim_end_matches('.').len();
result.push_str(&word[..lead]);
result.push_str(SECRET_MARK);
result.push_str(&word[word.len() - trail..]);
} else {
result.push_str(word)
}
word.clear();
};
for c in out.chars() {
@@ -402,6 +422,41 @@ mod tests {
let token = format!("{}.{}.{}", "M".repeat(24), "GabcdE", "x".repeat(30));
let out = redact(&format!("token={token}, ok id=123456789012345678 a.b.c"), &[]);
assert_eq!(out, "token=[비밀값], ok id=123456789012345678 a.b.c");
// Sentence dots around it do not hide it.
assert_eq!(redact(&format!("token={token}. next ...{token}..."), &[]), "token=[비밀값]. next ...[비밀값]...");
}
#[test]
fn the_zip_leaves_out_the_oldest_lines_by_time_across_tabs() {
let at_hm = |d: u32, h: u32, m: u32| Local.with_ymd_and_hms(2026, 5, d, h, m, 0).unwrap();
let dir = temp_dir("logorder");
let hub = LogHub::new(Some(&dir));
hub.push_at(at_hm(1, 0, 1), Level::Info, "chat", "first chat line");
hub.push_at(at_hm(1, 23, 59), Level::Info, "app", "late on the first");
hub.push_at(at_hm(2, 0, 1), Level::Info, "chat", "early on the second");
let mut files: Vec<(NaiveDate, String, Vec<String>)> = vec![];
for e in std::fs::read_dir(&dir).unwrap().flatten() {
let name = e.file_name().to_string_lossy().into_owned();
files.push((file_day(&name).unwrap(), name, std::fs::read_to_string(e.path()).unwrap().lines().map(str::to_owned).collect()));
}
files.sort();
// Leave out one line: the 00:01 chat line of the 1st, not a line of the app file.
let z = build_zip(&files, 1).unwrap();
let mut z = zip::ZipArchive::new(std::io::Cursor::new(z)).unwrap();
let mut all = String::new();
for i in 0..z.len() {
std::io::Read::read_to_string(&mut z.by_index(i).unwrap(), &mut all).unwrap();
}
assert!(!all.contains("first chat line"));
assert!(all.contains("late on the first") && all.contains("early on the second"));
// Two: then the 23:59 app line goes and the next day's chat line stays.
let z = build_zip(&files, 2).unwrap();
let mut z = zip::ZipArchive::new(std::io::Cursor::new(z)).unwrap();
let mut all = String::new();
for i in 0..z.len() {
std::io::Read::read_to_string(&mut z.by_index(i).unwrap(), &mut all).unwrap();
}
assert!(!all.contains("late on the first") && all.contains("early on the second"));
}
#[test]

View File

@@ -546,7 +546,8 @@ fn on_text(inner: &Inner, t: &str) {
inner.state.lock().unwrap().open_tts.remove(&d.id).then_some(WorkerEvent::TtsDone(d))
}),
"error" => serde_json::from_value::<WorkerError>(v).ok().map(|e| {
debug_log!("worker", "voice worker error {}", e.code);
// Always logged: the log screen and its error report must show what the worker refused.
error_log!("worker", "voice worker error {} id={}", e.code, e.id.as_deref().unwrap_or("-"));
// A request the worker refused outright never gets a tts_done.
if e.code == "tts_unavailable" || e.code == "brain_unavailable" {
if let Some(id) = &e.id {

View File

@@ -210,6 +210,25 @@ async fn a_refused_tts_request_ends_as_failed() {
c.close();
}
#[tokio::test]
async fn a_worker_error_reaches_the_log_hub_without_debug_logging() {
assert_ne!(std::env::var("JARVIS_DEBUG").as_deref(), Ok("1"), "run without debug logging");
let hub = jarvis_app::logs::LogHub::new(None);
jarvis_app::logs::install(hub.clone());
let hub = jarvis_app::logs::installed().unwrap();
let w = FakeWorker::start(0).await;
let (c, mut rx) = client(&w.url(), json!({}));
c.start();
c.wait_ready(Duration::from_secs(2)).await.unwrap();
w.send(Message::text(json!({"type": "error", "code": "tts_failed", "message": "engine broke", "id": "r7"}).to_string()));
next_matching(&mut rx, |e| matches!(e, WorkerEvent::Error(_))).await;
let errors = hub.query(Some("worker"), jarvis_app::logs::Level::Error, Some("tts_failed"), None, 10);
assert_eq!(errors.len(), 1, "{errors:?}");
assert!(errors[0].text.contains("r7"));
assert!(hub.recent_error_report("test").unwrap().contains("tts_failed"));
c.close();
}
#[tokio::test]
async fn reconnects_after_a_restart_and_fails_the_tts_in_flight() {
let w = FakeWorker::start(0).await;

View File

@@ -169,7 +169,7 @@ class JarvisBrain:
self._memories: dict = {}
self._prepared: dict[str, _Prepared] = {}
self._lock = threading.Lock()
debug_log(f"brain ready: llm={self.cfg.ollama_base_url} model={self.cfg.ollama_chat_model}", "voice_worker")
debug_log(f"brain ready: llm={self.cfg.ollama_base_url} model={self.cfg.ollama_chat_model}", "llm")
def _memory(self, user: str):
from jarvis.memory.conversation import DialogueMemory
@@ -198,7 +198,7 @@ class JarvisBrain:
finally:
p.done.set()
debug_log(f"brain prepare user={user} picked={p.tools is not None} "
f"ms={(time.perf_counter() - t0) * 1000:.0f}", "voice_worker")
f"ms={(time.perf_counter() - t0) * 1000:.0f}", "llm")
def _prepared_tools(self, user: str, text: str) -> Optional[list]:
"""The pick prepared for about this text, waiting for one still being made; None if none fits."""
@@ -233,7 +233,7 @@ class JarvisBrain:
answer = Muted.run(turn)
debug_log(f"brain reply user={user} ok={bool(answer)} prepared={prepared is not None} "
f"ms={(time.perf_counter() - t0) * 1000:.0f}", "voice_worker")
f"ms={(time.perf_counter() - t0) * 1000:.0f}", "llm")
return answer
def addressed(self, user: str, text: str, names: list[str]) -> Optional[bool]:

View File

@@ -50,7 +50,7 @@ def is_addressed(base_url: str, model: str, names: Sequence[str], text: str,
word = (raw or "").strip().lower().lstrip("\"'*").split()
answer = word[0].rstrip(".,!\"'*") if word else ""
result = True if answer == "yes" else False if answer == "no" else None
debug_log(f"judge addressed -> {result}", "voice_worker")
debug_log(f"judge addressed -> {result}", "llm")
return result
@@ -67,8 +67,8 @@ def rank_requests(base_url: str, model: str, texts: Sequence[str],
if 0 <= i < len(texts) and i not in order:
order.append(i)
if not order:
debug_log(f"judge rank candidates={len(texts)} -> none", "voice_worker")
debug_log(f"judge rank candidates={len(texts)} -> none", "llm")
return None
order += [i for i in range(len(texts)) if i not in order]
debug_log(f"judge rank candidates={len(texts)} -> first={order[0] + 1}", "voice_worker")
debug_log(f"judge rank candidates={len(texts)} -> first={order[0] + 1}", "llm")
return order

View File

@@ -120,9 +120,9 @@ class SttScheduler:
texts, error = [""] * len(batch), f"{type(e).__name__}: {e}"
ms = int((time.perf_counter() - t0) * 1000)
debug_log(f"stt batch kind={kind} size={len(batch)} beam={batch[0].opts.beam} ms={ms}"
+ (f" error={error}" if error else ""), "voice_worker")
+ (f" error={error}" if error else ""), "stt")
for job, text in zip(batch, texts):
try:
job.callback(SttResult(text, ms, error))
except Exception as e: # pragma: no cover - a broken session must not stop the loop
debug_log(f"stt callback failed: {type(e).__name__}: {e}", "voice_worker")
debug_log(f"stt callback failed: {type(e).__name__}: {e}", "stt")

View File

@@ -272,7 +272,7 @@ class Session:
self._partial_cb(speaker, st, ev.utt))
elif isinstance(ev, Final):
debug_log(f"utterance end speaker={speaker} utt={ev.utt} reason={ev.reason} "
f"ms={len(ev.audio) * 1000 // SR}", "voice_worker")
f"ms={len(ev.audio) * 1000 // SR}", "stt")
# Where, when and by whom it was said, fixed now: decoding can take a while, and a
# move or a late joiner meanwhile must not change who heard it.
said = (self.voice_channel, self.names.get(speaker, speaker),
@@ -546,7 +546,7 @@ class Session:
try:
await asyncio.to_thread(self.worker.brain.prepare, user, text, bool(self.chat_channels))
except Exception as e: # the brain must never take the session down
debug_log(f"brain prepare failed: {type(e).__name__}", "voice_worker")
debug_log(f"brain prepare failed: {type(e).__name__}", "llm")
newer, self.preparing[user] = self.preparing[user], None
text = newer if newer is not None and not similar(newer, self.last_prepared.get(user, "")) else None
finally:
@@ -565,9 +565,9 @@ class Session:
try:
yes = await asyncio.to_thread(self.worker.brain.addressed, user, text, list(self.settings.hotwords))
except Exception as e: # the brain must never take the session down
debug_log(f"brain addressed {qid} failed: {type(e).__name__}", "voice_worker")
debug_log(f"brain addressed {qid} failed: {type(e).__name__}", "llm")
yes = None
debug_log(f"brain addressed {qid} -> {yes}", "voice_worker")
debug_log(f"brain addressed {qid} -> {yes}", "llm")
self.send({"type": "addressed_result", "id": qid, "addressed": bool(yes), "failed": yes is None})
asyncio.get_running_loop().create_task(judge())
@@ -587,11 +587,11 @@ class Session:
try:
order = await asyncio.to_thread(self.worker.brain.rank, cands)
except Exception as e:
debug_log(f"brain rank {qid} failed: {type(e).__name__}", "voice_worker")
debug_log(f"brain rank {qid} failed: {type(e).__name__}", "llm")
order = None
known = {c["id"] for c in cands}
order = [i for i in (order or []) if i in known] if order is not None else None
debug_log(f"brain rank {qid} candidates={len(cands)} ok={order is not None}", "voice_worker")
debug_log(f"brain rank {qid} candidates={len(cands)} ok={order is not None}", "llm")
self.send({"type": "rank_result", "id": qid, "order": order or [], "failed": order is None})
asyncio.get_running_loop().create_task(rank())
@@ -622,7 +622,7 @@ class Session:
# They never got this answer, so nothing it asked can be confirmed next.
w = self.worker
w.reply_turns[user] = max(w.reply_turns.get(user, 0), turn + 1)
debug_log(f"brain reply {rid} did not reach the asker", "voice_worker")
debug_log(f"brain reply {rid} did not reach the asker", "llm")
self._reached(user)
def _on_reply_delivered(self, msg: dict) -> None:
@@ -653,14 +653,14 @@ class Session:
turn = w.reply_turns[user] = w.reply_turns.get(user, 0) + (2 if early else 1)
if early:
debug_log(f"brain reply {rid} arrived before the previous answer: a turn is skipped",
"voice_worker")
"llm")
from .chat_bridge import ChatBridge
chat = ChatBridge(self, asyncio.get_running_loop(), turn, w.pending_deletes.setdefault(user, {})) \
if self.chat_channels else None
try:
answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat)
except Exception as e: # the brain must never take the session down
debug_log(f"brain reply {rid} failed: {type(e).__name__}", "voice_worker")
debug_log(f"brain reply {rid} failed: {type(e).__name__}", "llm")
answer = None
answer = (answer or "").strip()
self.send({"type": "reply_text", "id": rid, "text": answer, "failed": not answer})
@@ -680,7 +680,7 @@ class Session:
if via == "voice" and channel and w.history is not None:
w.history.add_reply(channel, user, answer, time.time())
if speak and rid in self.cancelled_replies:
debug_log(f"brain reply {rid} not spoken: cancelled while answering", "voice_worker")
debug_log(f"brain reply {rid} not spoken: cancelled while answering", "llm")
self.send({"type": "tts_done", "id": rid, "chunks": 0, "cancelled": True, "failed": False})
elif speak:
self._on_tts({"type": "tts", "id": rid, "text": answer, "more": False})

View File

@@ -90,7 +90,7 @@ class WhisperTranscriber:
from faster_whisper import WhisperModel
self.model = WhisperModel(model, device=device, compute_type=compute_type)
self._tokenizers: dict = {}
debug_log(f"whisper loaded model={model} device={device} compute={compute_type}", "voice_worker")
debug_log(f"whisper loaded model={model} device={device} compute={compute_type}", "stt")
def _tokenizer(self, language: str):
tok = self._tokenizers.get(language)

View File

@@ -71,7 +71,7 @@ class TtsService:
def cancel(self, req: TtsRequest) -> None:
req.cancelled = True
req.closed = True
debug_log(f"tts cancel id={req.id} sent={req.chunks}", "voice_worker")
debug_log(f"tts cancel id={req.id} sent={req.chunks}", "tts")
self._maybe_done(req)
def _maybe_done(self, req: TtsRequest) -> None:
@@ -92,11 +92,11 @@ class TtsService:
except Exception as e:
req.failed = True
req.closed = True
debug_log(f"tts failed id={req.id} seq={seq}: {type(e).__name__}: {e}", "voice_worker")
debug_log(f"tts failed id={req.id} seq={seq}: {type(e).__name__}: {e}", "tts")
req.on_error(req, f"{type(e).__name__}: {e}")
continue
debug_log(f"tts sentence id={req.id} seq={seq} chars={len(text)} "
f"ms={int((time.perf_counter() - t0) * 1000)}", "voice_worker")
f"ms={int((time.perf_counter() - t0) * 1000)}", "tts")
if not req.cancelled:
req.chunks += 1
req.on_chunk(seq, pcm)

View File

@@ -32,7 +32,7 @@ class MeloEngine:
self.close()
raise RuntimeError(f"MeloTTS failed to start: {header.get('error')}")
self.native_rate = header["rate"]
debug_log(f"melo ready native_rate={self.native_rate}", "voice_worker")
debug_log(f"melo ready native_rate={self.native_rate}", "tts")
def _read(self) -> tuple[dict, bytes]:
out = self._proc.stdout
@@ -47,7 +47,7 @@ class MeloEngine:
def synthesize(self, text: str, rate: int) -> np.ndarray:
with self._lock:
if self._proc is None or self._proc.poll() is not None:
debug_log("melo child not running, restarting", "voice_worker")
debug_log("melo child not running, restarting", "tts")
self._start()
self._proc.stdin.write((json.dumps({"text": text, "rate": rate}) + "\n").encode())
self._proc.stdin.flush()

View File

@@ -311,6 +311,9 @@ never close the connection after the handshake.
## Debug logging
`debug_log(..., "voice_worker")` at: connection accepted or refused, settings changed, speaker left, utterance end
`debug_log(message, category)` at: connection accepted or refused, settings changed, speaker left, utterance end
(speaker, reason, length), each STT batch (kind, size, model time), each TTS sentence (length, time),
cancel, and engine failures.
cancel, and engine failures. The category names the service, so the app's log screen puts each line
in its tab (`app/src/admin.spec.md`, "Logs"): `stt` (utterance ends, STT batches, the Whisper model),
`tts` (sentences, cancels, MeloTTS), `llm` (the brain's replies, prepares and judgements),
`voice_worker` for the rest (connections, sessions, channels).

View File

@@ -1105,3 +1105,28 @@ def test_a_reply_waits_for_a_pick_still_being_prepared_for_its_text(tmp_path, mo
b.reply("100", "자비스 날씨 어때?", HistoryContext())
t.join()
assert seen["prepared"] == [plan]
@pytest.mark.unit
def test_worker_logs_name_their_service_so_the_app_puts_them_in_its_tab(store, monkeypatch, capfd):
"""The app's log screen puts a worker line in the stt, llm or tts tab by its category
(app/src/admin.spec.md, "Logs"); the worker must send those categories."""
from voice_worker import log as worker_log
monkeypatch.setattr(worker_log._jarvis_debug, "_is_debug_enabled", lambda: True)
def broken(user, text):
raise RuntimeError("model down")
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await say(ws, "100", 0.9)
await send(ws, type="reply", id="r1", user="100", text="hi", via="voice", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
await send(ws, type="tts", id="t1", text="안녕하세요.", more=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
asyncio.run(with_worker(body, store=store, brain=FakeBrain(answer=broken), tts=FakeTts()))
err = capfd.readouterr().err
for cat, what in (("stt", "utterance end"), ("llm", "brain reply r1 failed"), ("tts", "tts sentence id=t1")):
assert any(line.startswith(f"[{cat:^10}]") and what in line for line in err.splitlines()), (cat, err[-2000:])