fix(app): a worker run's late lines keep its run's secrets masked
Each worker run's log sink keeps the values masked when it started (its secrets in use), so a line still being logged after a stop gave up waiting, and after the next start replaced the values in use, is still masked. The debug echo on stderr is masked the same way. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -88,7 +88,9 @@ started again on the same port with the same token after 1 s, doubling to at mos
|
||||
failing (reset once a run printed `READY`). Stopping kills it; MeloTTS's child process ends when its
|
||||
input closes. A stop (and a restart) first logs everything the exited worker wrote, waiting at most
|
||||
5 s (a child of the worker may hold its output open): a stop returns only after its last lines were
|
||||
masked with the secrets that run was started with (`admin.spec.md`, "Logs").
|
||||
masked with the secrets that run was started with (`admin.spec.md`, "Logs"). Each run's lines are
|
||||
masked with the values masked when it started too, so a line still being logged after that wait
|
||||
(and after the next start replaced the values in use) stays masked.
|
||||
|
||||
## Worker link
|
||||
|
||||
@@ -227,7 +229,8 @@ registered until it is removed (M5 setup).
|
||||
silent at both ends, below its peak), speaker limit, chat access and splitting, settings, frames; the worker
|
||||
link against a fake server (hello, events, reconnect, failed TTS); the worker process with a
|
||||
stand-in `voice_worker` package run by a real Python (token only in the environment, restart on the
|
||||
same port, stop, the last output logged before a stop returns).
|
||||
same port, stop, the last output logged before a stop returns, a line logged after the stop gave up masked with its
|
||||
run's secrets).
|
||||
- `JARVIS_VOICE_PYTHON=<venv python> cargo test --test worker_protocol` runs the worker link and
|
||||
playback against the real Python worker server with fake engines (`tests/fixtures/fake_engine_worker.py`).
|
||||
- `tests/live_discord.rs` (opt-in, `JARVIS_LIVE_DISCORD=1`, see the file header): two bots in one
|
||||
|
||||
@@ -120,7 +120,18 @@ impl LogHub {
|
||||
|
||||
/// A line of the voice worker's output: its `[ category ]` prefix names the tab (stt, llm, tts,
|
||||
/// else worker); a Python warning or traceback line is a warning.
|
||||
/// Every value masked right now (stored and in use). A worker run keeps these for its lines
|
||||
/// (`push_worker_masked`), so a line it writes late is masked even after the next start.
|
||||
pub fn masked_values(&self) -> Vec<String> {
|
||||
self.state.lock().unwrap().masked()
|
||||
}
|
||||
|
||||
pub fn push_worker(&self, line: &str) {
|
||||
self.push_worker_masked(line, &[]);
|
||||
}
|
||||
|
||||
/// `push_worker`, also masking `extra` (the values of the run that wrote the line).
|
||||
pub fn push_worker_masked(&self, line: &str, extra: &[String]) {
|
||||
let (cat, text) = match line.strip_prefix('[').and_then(|r| r.split_once(']')) {
|
||||
Some((c, rest)) => (c.trim(), rest.trim_start()),
|
||||
None => ("worker", line),
|
||||
@@ -131,14 +142,20 @@ impl LogHub {
|
||||
};
|
||||
let level = if text.contains("Error") || text.contains("Traceback") { Level::Error }
|
||||
else if text.contains("Warning") { Level::Warn } else { Level::Info };
|
||||
self.push_at(Local::now(), level, tab, text);
|
||||
self.push_masked(Local::now(), level, tab, text, extra);
|
||||
}
|
||||
|
||||
fn push_at(&self, at: DateTime<Local>, level: Level, tab: &str, text: &str) {
|
||||
self.push_masked(at, level, tab, text, &[]);
|
||||
}
|
||||
|
||||
fn push_masked(&self, at: DateTime<Local>, level: Level, tab: &str, text: &str, extra: &[String]) {
|
||||
let mut s = self.state.lock().unwrap();
|
||||
// Masked before it is kept or written, so a secret replaced or removed later (the hub then
|
||||
// no longer knows the old value) never shows on the screen, in a copy, a file or an export.
|
||||
let text = redact(text, &s.masked());
|
||||
let mut masks = s.masked();
|
||||
masks.extend(extra.iter().cloned());
|
||||
let text = redact(text, &masks);
|
||||
let line = LogLine { seq: s.next_seq, at_ms: at.timestamp_millis(), level, tab: tab.into(), text };
|
||||
s.next_seq += 1;
|
||||
if let Some(dir) = s.dir.clone() {
|
||||
|
||||
@@ -27,7 +27,8 @@ impl std::fmt::Debug for LineSink {
|
||||
}
|
||||
|
||||
/// How long a stop or restart waits for the worker's last output to be logged. A child of the worker
|
||||
/// (an MCP server) may hold its output open after it is gone; then the rest is not read.
|
||||
/// (an MCP server) may hold its output open after it is gone; then the rest is not read. A line
|
||||
/// still being logged after that is masked with its run's values all the same (`run_sink`).
|
||||
const OUTPUT_DRAIN: Duration = Duration::from_secs(5);
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -124,6 +125,7 @@ impl WorkerProcess {
|
||||
pub fn start(&self) {
|
||||
let (opts, token, port) = (self.opts.clone(), self.token.clone(), self.port);
|
||||
let (shared, stop, events) = (self.shared.clone(), self.stop.clone(), self.events.clone());
|
||||
let sink = opts.line_sink.clone().unwrap_or_else(run_sink);
|
||||
let task = tokio::spawn(async move {
|
||||
let mut delay = opts.restart_delay;
|
||||
loop {
|
||||
@@ -143,7 +145,6 @@ impl WorkerProcess {
|
||||
shared.lock().unwrap().child_pid = child.id();
|
||||
crate::info_log!("worker", "voice worker started pid={:?} port={port}", child.id());
|
||||
let mut ready_rx = events.subscribe();
|
||||
let sink = opts.line_sink.clone().unwrap_or_else(|| LineSink(Arc::new(worker_line)));
|
||||
let mut readers = vec![];
|
||||
// The worker's own logs hold timings and counts only (voice_worker.spec.md).
|
||||
if let Some(out) = child.stdout.take() {
|
||||
@@ -161,6 +162,7 @@ impl WorkerProcess {
|
||||
}));
|
||||
}
|
||||
if let Some(err) = child.stderr.take() {
|
||||
let sink = sink.clone();
|
||||
readers.push(tokio::spawn(async move {
|
||||
let mut lines = BufReader::new(err).lines();
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
@@ -256,13 +258,21 @@ fn spawn(opts: &WorkerProcessOptions, token: &str, port: u16) -> std::io::Result
|
||||
cmd.spawn()
|
||||
}
|
||||
|
||||
/// The default sink of a worker started now: the app's log, masking also the values masked at
|
||||
/// this moment (this run's secrets, `logs.rs` `set_secrets_in_use`), so a line it writes late (after
|
||||
/// a stop gave up waiting for it, and the next start replaced the values in use) is still masked.
|
||||
pub fn run_sink() -> LineSink {
|
||||
let masks = crate::logs::installed().map(|h| h.masked_values()).unwrap_or_default();
|
||||
LineSink(Arc::new(move |line: &str| worker_line(line, &masks)))
|
||||
}
|
||||
|
||||
/// A line the voice worker printed: on stderr with debug logging, and always to the log hub (its
|
||||
/// own category picks the tab; warnings and tracebacks reach the log screen without debug logging).
|
||||
fn worker_line(line: &str) {
|
||||
fn worker_line(line: &str, masks: &[String]) {
|
||||
if crate::log::debug_enabled() {
|
||||
eprintln!("{}", crate::log::line("worker", line));
|
||||
eprintln!("{}", crate::log::line("worker", &crate::logs::redact(line, masks)));
|
||||
}
|
||||
if let Some(hub) = crate::logs::installed() {
|
||||
hub.push_worker(line);
|
||||
hub.push_worker_masked(line, masks);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,3 +150,51 @@ async fn stop_returns_only_after_the_last_output_is_logged() {
|
||||
let lines = seen.lock().unwrap().clone();
|
||||
assert!(lines.iter().any(|l| l.contains("LAST")), "{lines:?}");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn a_last_line_logged_after_the_stop_gave_up_is_masked_with_its_runs_secrets() {
|
||||
// The services started with the old key; the log of the last line is held up past the stop's
|
||||
// wait, and the next start replaces the values in use before that line is written.
|
||||
let hub = jarvis_app::logs::LogHub::new(None);
|
||||
jarvis_app::logs::install(hub.clone());
|
||||
hub.set_secrets_in_use(vec!["old-api-key-1".into()]);
|
||||
let fake = Fake::new("late");
|
||||
let mut o = fake.opts(false);
|
||||
o.env.insert("FAKE_LAST_LINES".into(), "1".into());
|
||||
let run = jarvis_app::worker_process::run_sink();
|
||||
let (started, done) = (std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)));
|
||||
let (s2, d2) = (started.clone(), done.clone());
|
||||
o.line_sink = Some(jarvis_app::worker_process::LineSink(std::sync::Arc::new(move |l: &str| {
|
||||
if l.contains("LAST") {
|
||||
s2.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
std::thread::sleep(Duration::from_secs(7));
|
||||
(run.0)(l);
|
||||
d2.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
} else {
|
||||
(run.0)(l);
|
||||
}
|
||||
})));
|
||||
let p = WorkerProcess::new(o).unwrap();
|
||||
let mut rx = p.subscribe();
|
||||
p.start();
|
||||
wait_for(&mut rx, |e| *e == ProcessEvent::Ready).await;
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while !started.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
}).await.expect("no last line");
|
||||
p.stop().await;
|
||||
// The next start: only the new key is in use (and stored).
|
||||
hub.set_secrets(vec!["new-api-key-2".into()]);
|
||||
hub.set_secrets_in_use(vec!["new-api-key-2".into()]);
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while !done.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
}).await.expect("the last line was never logged");
|
||||
let lines: Vec<String> = hub.query(None, jarvis_app::logs::Level::Debug, None, None, 100)
|
||||
.into_iter().map(|l| l.text).collect();
|
||||
let last = lines.iter().find(|l| l.contains("LAST")).expect("logged");
|
||||
assert!(last.contains("[비밀값]") && !last.contains("old-api-key-1"), "{lines:?}");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user