From eb0fa2f159fc542a90ca13ee4bee7fff0a13cd9b Mon Sep 17 00:00:00 2001 From: javis-bot Date: Thu, 12 Nov 2026 13:43:01 +0900 Subject: [PATCH] fix(app): log hub masks dotted tokens, keeps worker errors and drops the oldest lines by time - redact: a token with sentence dots around it is masked too. - Worker error messages are always logged (code and id), so the log screen and the error report show them without debug logging. - The voice worker logs with stt, llm and tts categories, so its lines land in those tabs. - The zip leaves out the oldest lines by the time they were logged, across every tab's file, not by file name. Co-Authored-By: Claude Opus 5.5 --- app/src/admin.spec.md | 7 ++- app/src/logs.rs | 71 ++++++++++++++++++++++++--- app/src/worker_client.rs | 3 +- app/tests/worker_client.rs | 19 +++++++ src/voice_worker/brain.py | 6 +-- src/voice_worker/judges.py | 6 +-- src/voice_worker/scheduler.py | 4 +- src/voice_worker/server.py | 20 ++++---- src/voice_worker/stt_whisper.py | 2 +- src/voice_worker/tts.py | 6 +-- src/voice_worker/tts_melo.py | 4 +- src/voice_worker/voice_worker.spec.md | 7 ++- tests/test_voice_worker_brain.py | 25 ++++++++++ 13 files changed, 143 insertions(+), 37 deletions(-) diff --git a/app/src/admin.spec.md b/app/src/admin.spec.md index fcb4943..19d5972 100644 --- a/app/src/admin.spec.md +++ b/app/src/admin.spec.md @@ -81,9 +81,12 @@ STT, LLM and TTS lines go to `stt`, `llm`, `tts` by their worker category), `bro order), the app version and a short PC summary (OS, CPU count, memory); `None` without an error. When it is over 2,000 characters (Discord's message limit) the screen offers a download instead. - `export_zip(from, to, out)`: the log files of those days in a zip, at most 10 MB (Discord's file - limit): when larger, the oldest lines are left out and a note says so. + limit): when larger, the oldest lines by the time they were logged, across every tab's file, are + left out and a note says how many. - Every copy and export runs through `redact`: each stored secret's value, and anything shaped like - a Discord bot token, becomes `[비밀값]`. + a Discord bot token (also with sentence dots around it), becomes `[비밀값]`. +- Errors the voice worker reports to the app (`error` messages: code and request id) are always + logged, debug logging or not, so the log screen and the error report show them. ## Tests diff --git a/app/src/logs.rs b/app/src/logs.rs index 523ec23..e1e0567 100644 --- a/app/src/logs.rs +++ b/app/src/logs.rs @@ -213,19 +213,25 @@ impl LogHub { } } -/// The zip with the oldest `dropped` lines (oldest files first) left out. +/// The zip with the `dropped` oldest lines left out: oldest by the time each line was logged, across +/// every tab's file (a line's time is its first 23 characters, `YYYY-MM-DD HH:MM:SS.mmm`; lines +/// without one count as oldest). fn build_zip(files: &[(NaiveDate, String, Vec)], dropped: usize) -> std::io::Result> { + let mut order: Vec<(&str, usize, usize)> = files.iter().enumerate() + .flat_map(|(fi, (_, _, lines))| lines.iter().enumerate() + .map(move |(li, l)| (l.get(..23).filter(|t| line_time_ok(t)).unwrap_or(""), fi, li))) + .collect(); + order.sort(); + let gone: std::collections::HashSet<(usize, usize)> = order.iter().take(dropped).map(|&(_, f, l)| (f, l)).collect(); let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); let opts = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); - let mut skip = dropped; - for (_, name, lines) in files { - let cut = skip.min(lines.len()); - skip -= cut; - if cut == lines.len() { + for (fi, (_, name, lines)) in files.iter().enumerate() { + let kept: Vec<&String> = lines.iter().enumerate().filter(|(li, _)| !gone.contains(&(fi, *li))).map(|(_, l)| l).collect(); + if kept.is_empty() { continue; } zip.start_file(name.as_str(), opts).map_err(std::io::Error::other)?; - for l in &lines[cut..] { + for l in kept { zip.write_all(l.as_bytes())?; zip.write_all(b"\n")?; } @@ -237,6 +243,10 @@ fn build_zip(files: &[(NaiveDate, String, Vec)], dropped: usize) -> std: Ok(zip.finish().map_err(std::io::Error::other)?.into_inner()) } +fn line_time_ok(t: &str) -> bool { + chrono::NaiveDateTime::parse_from_str(t, "%Y-%m-%d %H:%M:%S%.3f").is_ok() +} + fn file_day(name: &str) -> Option { let stem = name.strip_suffix(".log")?; let date = stem.get(stem.len().checked_sub(10)?..)?; @@ -269,8 +279,18 @@ pub fn redact(text: &str, secrets: &[String]) -> String { } let mut result = String::with_capacity(out.len()); let mut word = String::new(); + // A word may carry sentence dots around it ("token=."): they are not part of the token. let flush = |word: &mut String, result: &mut String| { - if looks_like_token(word) { result.push_str(SECRET_MARK) } else { result.push_str(word) } + let core = word.trim_matches('.'); + if looks_like_token(core) { + let lead = word.len() - word.trim_start_matches('.').len(); + let trail = word.len() - word.trim_end_matches('.').len(); + result.push_str(&word[..lead]); + result.push_str(SECRET_MARK); + result.push_str(&word[word.len() - trail..]); + } else { + result.push_str(word) + } word.clear(); }; for c in out.chars() { @@ -402,6 +422,41 @@ mod tests { let token = format!("{}.{}.{}", "M".repeat(24), "GabcdE", "x".repeat(30)); let out = redact(&format!("token={token}, ok id=123456789012345678 a.b.c"), &[]); assert_eq!(out, "token=[비밀값], ok id=123456789012345678 a.b.c"); + // Sentence dots around it do not hide it. + assert_eq!(redact(&format!("token={token}. next ...{token}..."), &[]), "token=[비밀값]. next ...[비밀값]..."); + } + + #[test] + fn the_zip_leaves_out_the_oldest_lines_by_time_across_tabs() { + let at_hm = |d: u32, h: u32, m: u32| Local.with_ymd_and_hms(2026, 5, d, h, m, 0).unwrap(); + let dir = temp_dir("logorder"); + let hub = LogHub::new(Some(&dir)); + hub.push_at(at_hm(1, 0, 1), Level::Info, "chat", "first chat line"); + hub.push_at(at_hm(1, 23, 59), Level::Info, "app", "late on the first"); + hub.push_at(at_hm(2, 0, 1), Level::Info, "chat", "early on the second"); + let mut files: Vec<(NaiveDate, String, Vec)> = vec![]; + for e in std::fs::read_dir(&dir).unwrap().flatten() { + let name = e.file_name().to_string_lossy().into_owned(); + files.push((file_day(&name).unwrap(), name, std::fs::read_to_string(e.path()).unwrap().lines().map(str::to_owned).collect())); + } + files.sort(); + // Leave out one line: the 00:01 chat line of the 1st, not a line of the app file. + let z = build_zip(&files, 1).unwrap(); + let mut z = zip::ZipArchive::new(std::io::Cursor::new(z)).unwrap(); + let mut all = String::new(); + for i in 0..z.len() { + std::io::Read::read_to_string(&mut z.by_index(i).unwrap(), &mut all).unwrap(); + } + assert!(!all.contains("first chat line")); + assert!(all.contains("late on the first") && all.contains("early on the second")); + // Two: then the 23:59 app line goes and the next day's chat line stays. + let z = build_zip(&files, 2).unwrap(); + let mut z = zip::ZipArchive::new(std::io::Cursor::new(z)).unwrap(); + let mut all = String::new(); + for i in 0..z.len() { + std::io::Read::read_to_string(&mut z.by_index(i).unwrap(), &mut all).unwrap(); + } + assert!(!all.contains("late on the first") && all.contains("early on the second")); } #[test] diff --git a/app/src/worker_client.rs b/app/src/worker_client.rs index 9634c01..9334be5 100644 --- a/app/src/worker_client.rs +++ b/app/src/worker_client.rs @@ -546,7 +546,8 @@ fn on_text(inner: &Inner, t: &str) { inner.state.lock().unwrap().open_tts.remove(&d.id).then_some(WorkerEvent::TtsDone(d)) }), "error" => serde_json::from_value::(v).ok().map(|e| { - debug_log!("worker", "voice worker error {}", e.code); + // Always logged: the log screen and its error report must show what the worker refused. + error_log!("worker", "voice worker error {} id={}", e.code, e.id.as_deref().unwrap_or("-")); // A request the worker refused outright never gets a tts_done. if e.code == "tts_unavailable" || e.code == "brain_unavailable" { if let Some(id) = &e.id { diff --git a/app/tests/worker_client.rs b/app/tests/worker_client.rs index 7fcce4b..cb759f2 100644 --- a/app/tests/worker_client.rs +++ b/app/tests/worker_client.rs @@ -210,6 +210,25 @@ async fn a_refused_tts_request_ends_as_failed() { c.close(); } +#[tokio::test] +async fn a_worker_error_reaches_the_log_hub_without_debug_logging() { + assert_ne!(std::env::var("JARVIS_DEBUG").as_deref(), Ok("1"), "run without debug logging"); + let hub = jarvis_app::logs::LogHub::new(None); + jarvis_app::logs::install(hub.clone()); + let hub = jarvis_app::logs::installed().unwrap(); + let w = FakeWorker::start(0).await; + let (c, mut rx) = client(&w.url(), json!({})); + c.start(); + c.wait_ready(Duration::from_secs(2)).await.unwrap(); + w.send(Message::text(json!({"type": "error", "code": "tts_failed", "message": "engine broke", "id": "r7"}).to_string())); + next_matching(&mut rx, |e| matches!(e, WorkerEvent::Error(_))).await; + let errors = hub.query(Some("worker"), jarvis_app::logs::Level::Error, Some("tts_failed"), None, 10); + assert_eq!(errors.len(), 1, "{errors:?}"); + assert!(errors[0].text.contains("r7")); + assert!(hub.recent_error_report("test").unwrap().contains("tts_failed")); + c.close(); +} + #[tokio::test] async fn reconnects_after_a_restart_and_fails_the_tts_in_flight() { let w = FakeWorker::start(0).await; diff --git a/src/voice_worker/brain.py b/src/voice_worker/brain.py index df49b5d..40887d1 100644 --- a/src/voice_worker/brain.py +++ b/src/voice_worker/brain.py @@ -169,7 +169,7 @@ class JarvisBrain: self._memories: dict = {} self._prepared: dict[str, _Prepared] = {} self._lock = threading.Lock() - debug_log(f"brain ready: llm={self.cfg.ollama_base_url} model={self.cfg.ollama_chat_model}", "voice_worker") + debug_log(f"brain ready: llm={self.cfg.ollama_base_url} model={self.cfg.ollama_chat_model}", "llm") def _memory(self, user: str): from jarvis.memory.conversation import DialogueMemory @@ -198,7 +198,7 @@ class JarvisBrain: finally: p.done.set() debug_log(f"brain prepare user={user} picked={p.tools is not None} " - f"ms={(time.perf_counter() - t0) * 1000:.0f}", "voice_worker") + f"ms={(time.perf_counter() - t0) * 1000:.0f}", "llm") def _prepared_tools(self, user: str, text: str) -> Optional[list]: """The pick prepared for about this text, waiting for one still being made; None if none fits.""" @@ -233,7 +233,7 @@ class JarvisBrain: answer = Muted.run(turn) debug_log(f"brain reply user={user} ok={bool(answer)} prepared={prepared is not None} " - f"ms={(time.perf_counter() - t0) * 1000:.0f}", "voice_worker") + f"ms={(time.perf_counter() - t0) * 1000:.0f}", "llm") return answer def addressed(self, user: str, text: str, names: list[str]) -> Optional[bool]: diff --git a/src/voice_worker/judges.py b/src/voice_worker/judges.py index 95b2a6a..728f536 100644 --- a/src/voice_worker/judges.py +++ b/src/voice_worker/judges.py @@ -50,7 +50,7 @@ def is_addressed(base_url: str, model: str, names: Sequence[str], text: str, word = (raw or "").strip().lower().lstrip("\"'*").split() answer = word[0].rstrip(".,!\"'*") if word else "" result = True if answer == "yes" else False if answer == "no" else None - debug_log(f"judge addressed -> {result}", "voice_worker") + debug_log(f"judge addressed -> {result}", "llm") return result @@ -67,8 +67,8 @@ def rank_requests(base_url: str, model: str, texts: Sequence[str], if 0 <= i < len(texts) and i not in order: order.append(i) if not order: - debug_log(f"judge rank candidates={len(texts)} -> none", "voice_worker") + debug_log(f"judge rank candidates={len(texts)} -> none", "llm") return None order += [i for i in range(len(texts)) if i not in order] - debug_log(f"judge rank candidates={len(texts)} -> first={order[0] + 1}", "voice_worker") + debug_log(f"judge rank candidates={len(texts)} -> first={order[0] + 1}", "llm") return order diff --git a/src/voice_worker/scheduler.py b/src/voice_worker/scheduler.py index 1042f15..7204f6a 100644 --- a/src/voice_worker/scheduler.py +++ b/src/voice_worker/scheduler.py @@ -120,9 +120,9 @@ class SttScheduler: texts, error = [""] * len(batch), f"{type(e).__name__}: {e}" ms = int((time.perf_counter() - t0) * 1000) debug_log(f"stt batch kind={kind} size={len(batch)} beam={batch[0].opts.beam} ms={ms}" - + (f" error={error}" if error else ""), "voice_worker") + + (f" error={error}" if error else ""), "stt") for job, text in zip(batch, texts): try: job.callback(SttResult(text, ms, error)) except Exception as e: # pragma: no cover - a broken session must not stop the loop - debug_log(f"stt callback failed: {type(e).__name__}: {e}", "voice_worker") + debug_log(f"stt callback failed: {type(e).__name__}: {e}", "stt") diff --git a/src/voice_worker/server.py b/src/voice_worker/server.py index 3f9bbc5..ff5732d 100644 --- a/src/voice_worker/server.py +++ b/src/voice_worker/server.py @@ -272,7 +272,7 @@ class Session: self._partial_cb(speaker, st, ev.utt)) elif isinstance(ev, Final): debug_log(f"utterance end speaker={speaker} utt={ev.utt} reason={ev.reason} " - f"ms={len(ev.audio) * 1000 // SR}", "voice_worker") + f"ms={len(ev.audio) * 1000 // SR}", "stt") # Where, when and by whom it was said, fixed now: decoding can take a while, and a # move or a late joiner meanwhile must not change who heard it. said = (self.voice_channel, self.names.get(speaker, speaker), @@ -546,7 +546,7 @@ class Session: try: await asyncio.to_thread(self.worker.brain.prepare, user, text, bool(self.chat_channels)) except Exception as e: # the brain must never take the session down - debug_log(f"brain prepare failed: {type(e).__name__}", "voice_worker") + debug_log(f"brain prepare failed: {type(e).__name__}", "llm") newer, self.preparing[user] = self.preparing[user], None text = newer if newer is not None and not similar(newer, self.last_prepared.get(user, "")) else None finally: @@ -565,9 +565,9 @@ class Session: try: yes = await asyncio.to_thread(self.worker.brain.addressed, user, text, list(self.settings.hotwords)) except Exception as e: # the brain must never take the session down - debug_log(f"brain addressed {qid} failed: {type(e).__name__}", "voice_worker") + debug_log(f"brain addressed {qid} failed: {type(e).__name__}", "llm") yes = None - debug_log(f"brain addressed {qid} -> {yes}", "voice_worker") + debug_log(f"brain addressed {qid} -> {yes}", "llm") self.send({"type": "addressed_result", "id": qid, "addressed": bool(yes), "failed": yes is None}) asyncio.get_running_loop().create_task(judge()) @@ -587,11 +587,11 @@ class Session: try: order = await asyncio.to_thread(self.worker.brain.rank, cands) except Exception as e: - debug_log(f"brain rank {qid} failed: {type(e).__name__}", "voice_worker") + debug_log(f"brain rank {qid} failed: {type(e).__name__}", "llm") order = None known = {c["id"] for c in cands} order = [i for i in (order or []) if i in known] if order is not None else None - debug_log(f"brain rank {qid} candidates={len(cands)} ok={order is not None}", "voice_worker") + debug_log(f"brain rank {qid} candidates={len(cands)} ok={order is not None}", "llm") self.send({"type": "rank_result", "id": qid, "order": order or [], "failed": order is None}) asyncio.get_running_loop().create_task(rank()) @@ -622,7 +622,7 @@ class Session: # They never got this answer, so nothing it asked can be confirmed next. w = self.worker w.reply_turns[user] = max(w.reply_turns.get(user, 0), turn + 1) - debug_log(f"brain reply {rid} did not reach the asker", "voice_worker") + debug_log(f"brain reply {rid} did not reach the asker", "llm") self._reached(user) def _on_reply_delivered(self, msg: dict) -> None: @@ -653,14 +653,14 @@ class Session: turn = w.reply_turns[user] = w.reply_turns.get(user, 0) + (2 if early else 1) if early: debug_log(f"brain reply {rid} arrived before the previous answer: a turn is skipped", - "voice_worker") + "llm") from .chat_bridge import ChatBridge chat = ChatBridge(self, asyncio.get_running_loop(), turn, w.pending_deletes.setdefault(user, {})) \ if self.chat_channels else None try: answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat) except Exception as e: # the brain must never take the session down - debug_log(f"brain reply {rid} failed: {type(e).__name__}", "voice_worker") + debug_log(f"brain reply {rid} failed: {type(e).__name__}", "llm") answer = None answer = (answer or "").strip() self.send({"type": "reply_text", "id": rid, "text": answer, "failed": not answer}) @@ -680,7 +680,7 @@ class Session: if via == "voice" and channel and w.history is not None: w.history.add_reply(channel, user, answer, time.time()) if speak and rid in self.cancelled_replies: - debug_log(f"brain reply {rid} not spoken: cancelled while answering", "voice_worker") + debug_log(f"brain reply {rid} not spoken: cancelled while answering", "llm") self.send({"type": "tts_done", "id": rid, "chunks": 0, "cancelled": True, "failed": False}) elif speak: self._on_tts({"type": "tts", "id": rid, "text": answer, "more": False}) diff --git a/src/voice_worker/stt_whisper.py b/src/voice_worker/stt_whisper.py index 047fc52..2b1ad67 100644 --- a/src/voice_worker/stt_whisper.py +++ b/src/voice_worker/stt_whisper.py @@ -90,7 +90,7 @@ class WhisperTranscriber: from faster_whisper import WhisperModel self.model = WhisperModel(model, device=device, compute_type=compute_type) self._tokenizers: dict = {} - debug_log(f"whisper loaded model={model} device={device} compute={compute_type}", "voice_worker") + debug_log(f"whisper loaded model={model} device={device} compute={compute_type}", "stt") def _tokenizer(self, language: str): tok = self._tokenizers.get(language) diff --git a/src/voice_worker/tts.py b/src/voice_worker/tts.py index 1613596..ca40bfd 100644 --- a/src/voice_worker/tts.py +++ b/src/voice_worker/tts.py @@ -71,7 +71,7 @@ class TtsService: def cancel(self, req: TtsRequest) -> None: req.cancelled = True req.closed = True - debug_log(f"tts cancel id={req.id} sent={req.chunks}", "voice_worker") + debug_log(f"tts cancel id={req.id} sent={req.chunks}", "tts") self._maybe_done(req) def _maybe_done(self, req: TtsRequest) -> None: @@ -92,11 +92,11 @@ class TtsService: except Exception as e: req.failed = True req.closed = True - debug_log(f"tts failed id={req.id} seq={seq}: {type(e).__name__}: {e}", "voice_worker") + debug_log(f"tts failed id={req.id} seq={seq}: {type(e).__name__}: {e}", "tts") req.on_error(req, f"{type(e).__name__}: {e}") continue debug_log(f"tts sentence id={req.id} seq={seq} chars={len(text)} " - f"ms={int((time.perf_counter() - t0) * 1000)}", "voice_worker") + f"ms={int((time.perf_counter() - t0) * 1000)}", "tts") if not req.cancelled: req.chunks += 1 req.on_chunk(seq, pcm) diff --git a/src/voice_worker/tts_melo.py b/src/voice_worker/tts_melo.py index bd506f8..a744cda 100644 --- a/src/voice_worker/tts_melo.py +++ b/src/voice_worker/tts_melo.py @@ -32,7 +32,7 @@ class MeloEngine: self.close() raise RuntimeError(f"MeloTTS failed to start: {header.get('error')}") self.native_rate = header["rate"] - debug_log(f"melo ready native_rate={self.native_rate}", "voice_worker") + debug_log(f"melo ready native_rate={self.native_rate}", "tts") def _read(self) -> tuple[dict, bytes]: out = self._proc.stdout @@ -47,7 +47,7 @@ class MeloEngine: def synthesize(self, text: str, rate: int) -> np.ndarray: with self._lock: if self._proc is None or self._proc.poll() is not None: - debug_log("melo child not running, restarting", "voice_worker") + debug_log("melo child not running, restarting", "tts") self._start() self._proc.stdin.write((json.dumps({"text": text, "rate": rate}) + "\n").encode()) self._proc.stdin.flush() diff --git a/src/voice_worker/voice_worker.spec.md b/src/voice_worker/voice_worker.spec.md index eaff6dc..2d80d4a 100644 --- a/src/voice_worker/voice_worker.spec.md +++ b/src/voice_worker/voice_worker.spec.md @@ -311,6 +311,9 @@ never close the connection after the handshake. ## Debug logging -`debug_log(..., "voice_worker")` at: connection accepted or refused, settings changed, speaker left, utterance end +`debug_log(message, category)` at: connection accepted or refused, settings changed, speaker left, utterance end (speaker, reason, length), each STT batch (kind, size, model time), each TTS sentence (length, time), -cancel, and engine failures. +cancel, and engine failures. The category names the service, so the app's log screen puts each line +in its tab (`app/src/admin.spec.md`, "Logs"): `stt` (utterance ends, STT batches, the Whisper model), +`tts` (sentences, cancels, MeloTTS), `llm` (the brain's replies, prepares and judgements), +`voice_worker` for the rest (connections, sessions, channels). diff --git a/tests/test_voice_worker_brain.py b/tests/test_voice_worker_brain.py index 4d560c8..aadbd3c 100644 --- a/tests/test_voice_worker_brain.py +++ b/tests/test_voice_worker_brain.py @@ -1105,3 +1105,28 @@ def test_a_reply_waits_for_a_pick_still_being_prepared_for_its_text(tmp_path, mo b.reply("100", "자비스 날씨 어때?", HistoryContext()) t.join() assert seen["prepared"] == [plan] + + +@pytest.mark.unit +def test_worker_logs_name_their_service_so_the_app_puts_them_in_its_tab(store, monkeypatch, capfd): + """The app's log screen puts a worker line in the stt, llm or tts tab by its category + (app/src/admin.spec.md, "Logs"); the worker must send those categories.""" + from voice_worker import log as worker_log + monkeypatch.setattr(worker_log._jarvis_debug, "_is_debug_enabled", lambda: True) + + def broken(user, text): + raise RuntimeError("model down") + + async def body(url): + async with connect(url) as ws: + await hello(ws, stt={"partials": False}) + await say(ws, "100", 0.9) + await send(ws, type="reply", id="r1", user="100", text="hi", via="voice", speak=False) + await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text") + await send(ws, type="tts", id="t1", text="안녕하세요.", more=False) + await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done") + + asyncio.run(with_worker(body, store=store, brain=FakeBrain(answer=broken), tts=FakeTts())) + err = capfd.readouterr().err + for cat, what in (("stt", "utterance end"), ("llm", "brain reply r1 failed"), ("tts", "tts sentence id=t1")): + assert any(line.startswith(f"[{cat:^10}]") and what in line for line in err.splitlines()), (cat, err[-2000:])