//! Starts the Python voice worker as a child process and restarts it if it dies. //! //! Each app launch makes a new random token and hands it over in the environment //! (`JARVIS_VOICE_TOKEN`), never on the command line. The worker listens on 127.0.0.1 only. use std::collections::HashMap; use std::path::PathBuf; use std::process::Stdio; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use base64::Engine; use tokio::io::{AsyncBufReadExt, BufReader}; use tokio::process::{Child, Command}; use tokio::sync::{broadcast, Notify}; use crate::error_log; /// Where the worker's output lines go (default: the app's log, `worker_line`). #[derive(Clone)] pub struct LineSink(pub Arc); impl std::fmt::Debug for LineSink { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("LineSink") } } /// How long a stop or restart waits for the worker's last output to be logged. A child of the worker /// (an MCP server) may hold its output open after it is gone; then the rest is not read. A line /// still being logged after that is masked with its run's values all the same (`run_sink`). const OUTPUT_DRAIN: Duration = Duration::from_secs(5); #[derive(Debug, Clone)] pub struct WorkerProcessOptions { pub python: String, pub cwd: PathBuf, pub args: Vec, /// Fixed port; 0 picks a free one once at start (kept across restarts). pub port: u16, /// Extra environment for the worker (over the app's own). pub env: HashMap, pub restart_delay: Duration, pub max_restart_delay: Duration, /// Where output lines go; None: the app's log. pub line_sink: Option, } impl WorkerProcessOptions { pub fn new(python: impl Into, cwd: impl Into, args: Vec) -> Self { Self { python: python.into(), cwd: cwd.into(), args, port: 0, env: HashMap::new(), restart_delay: Duration::from_secs(1), max_restart_delay: Duration::from_secs(30), line_sink: None, } } } #[derive(Debug, Clone, PartialEq, Eq)] pub enum ProcessEvent { /// The worker printed READY: models are loaded and it accepts connections. Ready, Exited { code: Option }, } pub fn free_port() -> std::io::Result { Ok(std::net::TcpListener::bind("127.0.0.1:0")?.local_addr()?.port()) } struct Shared { stopping: bool, child_pid: Option, restarts: u32, } pub struct WorkerProcess { opts: WorkerProcessOptions, pub token: String, port: u16, shared: Arc>, stop: Arc, events: broadcast::Sender, task: Mutex>>, } impl WorkerProcess { pub fn new(opts: WorkerProcessOptions) -> std::io::Result { let port = if opts.port == 0 { free_port()? } else { opts.port }; let mut raw = [0u8; 32]; rand::fill(&mut raw); let token = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(raw); Ok(Self { opts, token, port, shared: Arc::new(Mutex::new(Shared { stopping: false, child_pid: None, restarts: 0 })), stop: Arc::new(Notify::new()), events: broadcast::channel(64).0, task: Mutex::new(None), }) } pub fn url(&self) -> String { format!("ws://127.0.0.1:{}", self.port) } pub fn port(&self) -> u16 { self.port } pub fn restarts(&self) -> u32 { self.shared.lock().unwrap().restarts } pub fn subscribe(&self) -> broadcast::Receiver { self.events.subscribe() } /// Spawn the worker and keep it running until `stop()`. pub fn start(&self) { let (opts, token, port) = (self.opts.clone(), self.token.clone(), self.port); let (shared, stop, events) = (self.shared.clone(), self.stop.clone(), self.events.clone()); let sink = opts.line_sink.clone().unwrap_or_else(run_sink); let task = tokio::spawn(async move { let mut delay = opts.restart_delay; loop { let started = Instant::now(); let mut child = match spawn(&opts, &token, port) { Ok(c) => c, Err(e) => { error_log!("worker", "voice worker could not start: {e}"); let _ = events.send(ProcessEvent::Exited { code: None }); if wait_or_stop(&stop, &shared, delay).await { return; } delay = (delay * 2).min(opts.max_restart_delay); continue; } }; shared.lock().unwrap().child_pid = child.id(); crate::info_log!("worker", "voice worker started pid={:?} port={port}", child.id()); let mut ready_rx = events.subscribe(); let mut readers = vec![]; // The worker's own logs hold timings and counts only (voice_worker.spec.md). if let Some(out) = child.stdout.take() { let (events, sink) = (events.clone(), sink.clone()); readers.push(tokio::spawn(async move { let mut lines = BufReader::new(out).lines(); while let Ok(Some(line)) = lines.next_line().await { if line.starts_with("READY ") { crate::info_log!("worker", "voice worker READY after {} ms", started.elapsed().as_millis()); let _ = events.send(ProcessEvent::Ready); } else { (sink.0)(&line); } } })); } if let Some(err) = child.stderr.take() { let sink = sink.clone(); readers.push(tokio::spawn(async move { let mut lines = BufReader::new(err).lines(); while let Ok(Some(line)) = lines.next_line().await { (sink.0)(&line); } })); } let status = tokio::select! { s = child.wait() => s.ok(), _ = stop.notified() => { let _ = child.kill().await; let code = child.wait().await.ok().and_then(|s| s.code()); drain(readers).await; let _ = events.send(ProcessEvent::Exited { code }); return; } }; // A worker that got as far as READY resets the backoff. while let Ok(ev) = ready_rx.try_recv() { if ev == ProcessEvent::Ready { delay = opts.restart_delay; } } let code = status.and_then(|s| s.code()); drain(readers).await; shared.lock().unwrap().child_pid = None; let _ = events.send(ProcessEvent::Exited { code }); if shared.lock().unwrap().stopping { return; } error_log!("worker", "voice worker exited (code {code:?}); restarting in {} ms", delay.as_millis()); if wait_or_stop(&stop, &shared, delay).await { return; } shared.lock().unwrap().restarts += 1; delay = (delay * 2).min(opts.max_restart_delay); } }); *self.task.lock().unwrap() = Some(task); } /// Stop the worker and do not restart it. Returns once it has exited and its last output is logged. pub async fn stop(&self) { self.shared.lock().unwrap().stopping = true; self.stop.notify_one(); let task = self.task.lock().unwrap().take(); if let Some(t) = task { let _ = t.await; } } } /// Wait until the output readers have logged everything the exited worker wrote (at most /// `OUTPUT_DRAIN`), so a stop returns only after its last lines were masked with this run's secrets /// (`logs.rs`, `set_secrets_in_use`); readers still waiting after that are stopped. async fn drain(readers: Vec>) { let until = tokio::time::Instant::now() + OUTPUT_DRAIN; for mut r in readers { if tokio::time::timeout_at(until, &mut r).await.is_err() { r.abort(); } } } /// Sleep `delay` unless stopped meanwhile (true = stopped). async fn wait_or_stop(stop: &Notify, shared: &Mutex, delay: Duration) -> bool { if shared.lock().unwrap().stopping { return true; } tokio::select! { _ = tokio::time::sleep(delay) => shared.lock().unwrap().stopping, _ = stop.notified() => true, } } fn spawn(opts: &WorkerProcessOptions, token: &str, port: u16) -> std::io::Result { let mut cmd = Command::new(&opts.python); cmd.args(["-m", "voice_worker", "--host", "127.0.0.1", "--port", &port.to_string()]) .args(&opts.args) .current_dir(&opts.cwd) .envs(&opts.env) .env("JARVIS_VOICE_TOKEN", token) .env("PYTHONUNBUFFERED", "1") .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .kill_on_drop(true); #[cfg(windows)] { const CREATE_NO_WINDOW: u32 = 0x0800_0000; cmd.creation_flags(CREATE_NO_WINDOW); } cmd.spawn() } /// The default sink of a worker started now: the app's log, masking also the values masked at /// this moment (this run's secrets, `logs.rs` `set_secrets_in_use`), so a line it writes late (after /// a stop gave up waiting for it, and the next start replaced the values in use) is still masked. pub fn run_sink() -> LineSink { let masks = crate::logs::installed().map(|h| h.masked_values()).unwrap_or_default(); LineSink(Arc::new(move |line: &str| worker_line(line, &masks))) } /// A line the voice worker printed: on stderr with debug logging, and always to the log hub (its /// own category picks the tab; warnings and tracebacks reach the log screen without debug logging). fn worker_line(line: &str, masks: &[String]) { if crate::log::debug_enabled() { eprintln!("{}", crate::log::line("worker", &crate::logs::redact(line, masks))); } if let Some(hub) = crate::logs::installed() { hub.push_worker_masked(line, masks); } }