diff --git a/app/src/discord_core.spec.md b/app/src/discord_core.spec.md index db522e9..0ed2628 100644 --- a/app/src/discord_core.spec.md +++ b/app/src/discord_core.spec.md @@ -86,7 +86,9 @@ The app starts `python -m voice_worker --host 127.0.0.1 --port ` in `JARVIS_VOICE_TOKEN`. The port is a free loopback port chosen once at start. If the worker exits, it is started again on the same port with the same token after 1 s, doubling to at most 30 s while it keeps failing (reset once a run printed `READY`). Stopping kills it; MeloTTS's child process ends when its -input closes. +input closes. A stop (and a restart) first logs everything the exited worker wrote, waiting at most +5 s (a child of the worker may hold its output open): a stop returns only after its last lines were +masked with the secrets that run was started with (`admin.spec.md`, "Logs"). ## Worker link @@ -225,7 +227,7 @@ registered until it is removed (M5 setup). silent at both ends, below its peak), speaker limit, chat access and splitting, settings, frames; the worker link against a fake server (hello, events, reconnect, failed TTS); the worker process with a stand-in `voice_worker` package run by a real Python (token only in the environment, restart on the - same port, stop). + same port, stop, the last output logged before a stop returns). - `JARVIS_VOICE_PYTHON= cargo test --test worker_protocol` runs the worker link and playback against the real Python worker server with fake engines (`tests/fixtures/fake_engine_worker.py`). - `tests/live_discord.rs` (opt-in, `JARVIS_LIVE_DISCORD=1`, see the file header): two bots in one diff --git a/app/src/worker_process.rs b/app/src/worker_process.rs index a4b1033..ee89e4e 100644 --- a/app/src/worker_process.rs +++ b/app/src/worker_process.rs @@ -16,6 +16,20 @@ use tokio::sync::{broadcast, Notify}; use crate::error_log; +/// Where the worker's output lines go (default: the app's log, `worker_line`). +#[derive(Clone)] +pub struct LineSink(pub Arc); + +impl std::fmt::Debug for LineSink { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("LineSink") + } +} + +/// How long a stop or restart waits for the worker's last output to be logged. A child of the worker +/// (an MCP server) may hold its output open after it is gone; then the rest is not read. +const OUTPUT_DRAIN: Duration = Duration::from_secs(5); + #[derive(Debug, Clone)] pub struct WorkerProcessOptions { pub python: String, @@ -27,6 +41,8 @@ pub struct WorkerProcessOptions { pub env: HashMap, pub restart_delay: Duration, pub max_restart_delay: Duration, + /// Where output lines go; None: the app's log. + pub line_sink: Option, } impl WorkerProcessOptions { @@ -39,6 +55,7 @@ impl WorkerProcessOptions { env: HashMap::new(), restart_delay: Duration::from_secs(1), max_restart_delay: Duration::from_secs(30), + line_sink: None, } } } @@ -126,34 +143,37 @@ impl WorkerProcess { shared.lock().unwrap().child_pid = child.id(); crate::info_log!("worker", "voice worker started pid={:?} port={port}", child.id()); let mut ready_rx = events.subscribe(); + let sink = opts.line_sink.clone().unwrap_or_else(|| LineSink(Arc::new(worker_line))); + let mut readers = vec![]; // The worker's own logs hold timings and counts only (voice_worker.spec.md). if let Some(out) = child.stdout.take() { - let events = events.clone(); - tokio::spawn(async move { + let (events, sink) = (events.clone(), sink.clone()); + readers.push(tokio::spawn(async move { let mut lines = BufReader::new(out).lines(); while let Ok(Some(line)) = lines.next_line().await { if line.starts_with("READY ") { crate::info_log!("worker", "voice worker READY after {} ms", started.elapsed().as_millis()); let _ = events.send(ProcessEvent::Ready); } else { - worker_line(&line); + (sink.0)(&line); } } - }); + })); } if let Some(err) = child.stderr.take() { - tokio::spawn(async move { + readers.push(tokio::spawn(async move { let mut lines = BufReader::new(err).lines(); while let Ok(Some(line)) = lines.next_line().await { - worker_line(&line); + (sink.0)(&line); } - }); + })); } let status = tokio::select! { s = child.wait() => s.ok(), _ = stop.notified() => { let _ = child.kill().await; let code = child.wait().await.ok().and_then(|s| s.code()); + drain(readers).await; let _ = events.send(ProcessEvent::Exited { code }); return; } @@ -165,6 +185,7 @@ impl WorkerProcess { } } let code = status.and_then(|s| s.code()); + drain(readers).await; shared.lock().unwrap().child_pid = None; let _ = events.send(ProcessEvent::Exited { code }); if shared.lock().unwrap().stopping { @@ -181,7 +202,7 @@ impl WorkerProcess { *self.task.lock().unwrap() = Some(task); } - /// Stop the worker and do not restart it. Returns once it has exited. + /// Stop the worker and do not restart it. Returns once it has exited and its last output is logged. pub async fn stop(&self) { self.shared.lock().unwrap().stopping = true; self.stop.notify_one(); @@ -192,6 +213,18 @@ impl WorkerProcess { } } +/// Wait until the output readers have logged everything the exited worker wrote (at most +/// `OUTPUT_DRAIN`), so a stop returns only after its last lines were masked with this run's secrets +/// (`logs.rs`, `set_secrets_in_use`); readers still waiting after that are stopped. +async fn drain(readers: Vec>) { + let until = tokio::time::Instant::now() + OUTPUT_DRAIN; + for mut r in readers { + if tokio::time::timeout_at(until, &mut r).await.is_err() { + r.abort(); + } + } +} + /// Sleep `delay` unless stopped meanwhile (true = stopped). async fn wait_or_stop(stop: &Notify, shared: &Mutex, delay: Duration) -> bool { if shared.lock().unwrap().stopping { diff --git a/app/tests/worker_process.rs b/app/tests/worker_process.rs index 45bb44b..43e8cac 100644 --- a/app/tests/worker_process.rs +++ b/app/tests/worker_process.rs @@ -17,6 +17,10 @@ if os.environ.get("FAKE_CRASH_FIRST") == "1" and runs == 0: sys.exit(3) print("loading models...", flush=True) print("READY 127.0.0.1:" + sys.argv[sys.argv.index("--port") + 1], flush=True) +if os.environ.get("FAKE_LAST_LINES") == "1": + for i in range(3): + print(f"line {i}", file=sys.stderr, flush=True) + print("LAST bad key old-api-key-1", file=sys.stderr, flush=True) time.sleep(60) "#; @@ -118,3 +122,31 @@ async fn stop_ends_the_worker_for_good() { assert!(rx.try_recv().is_err(), "no restart after stop"); assert_eq!(p.restarts(), 0); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stop_returns_only_after_the_last_output_is_logged() { + // A slow log (each line takes 300 ms) and a worker whose last lines are still unread when it is + // stopped: they must be handled before stop returns, so the next start's secrets (logs.rs, + // set_secrets_in_use) never apply to them. + let fake = Fake::new("drain"); + let mut o = fake.opts(false); + o.env.insert("FAKE_LAST_LINES".into(), "1".into()); + let seen = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let s2 = seen.clone(); + o.line_sink = Some(jarvis_app::worker_process::LineSink(std::sync::Arc::new(move |l: &str| { + std::thread::sleep(Duration::from_millis(300)); + s2.lock().unwrap().push(l.to_owned()); + }))); + let p = WorkerProcess::new(o).unwrap(); + let mut rx = p.subscribe(); + p.start(); + wait_for(&mut rx, |e| *e == ProcessEvent::Ready).await; + tokio::time::timeout(Duration::from_secs(10), async { + while seen.lock().unwrap().is_empty() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }).await.expect("no output"); + p.stop().await; + let lines = seen.lock().unwrap().clone(); + assert!(lines.iter().any(|l| l.contains("LAST")), "{lines:?}"); +}