Each worker run's log sink keeps the values masked when it started (its secrets in use), so a line still being logged after a stop gave up waiting, and after the next start replaced the values in use, is still masked. The debug echo on stderr is masked the same way. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
279 lines
10 KiB
Rust
279 lines
10 KiB
Rust
//! Starts the Python voice worker as a child process and restarts it if it dies.
|
|
//!
|
|
//! Each app launch makes a new random token and hands it over in the environment
|
|
//! (`JARVIS_VOICE_TOKEN`), never on the command line. The worker listens on 127.0.0.1 only.
|
|
|
|
use std::collections::HashMap;
|
|
use std::path::PathBuf;
|
|
use std::process::Stdio;
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::{Duration, Instant};
|
|
|
|
use base64::Engine;
|
|
use tokio::io::{AsyncBufReadExt, BufReader};
|
|
use tokio::process::{Child, Command};
|
|
use tokio::sync::{broadcast, Notify};
|
|
|
|
use crate::error_log;
|
|
|
|
/// Where the worker's output lines go (default: the app's log, `worker_line`).
|
|
#[derive(Clone)]
|
|
pub struct LineSink(pub Arc<dyn Fn(&str) + Send + Sync>);
|
|
|
|
impl std::fmt::Debug for LineSink {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
f.write_str("LineSink")
|
|
}
|
|
}
|
|
|
|
/// How long a stop or restart waits for the worker's last output to be logged. A child of the worker
|
|
/// (an MCP server) may hold its output open after it is gone; then the rest is not read. A line
|
|
/// still being logged after that is masked with its run's values all the same (`run_sink`).
|
|
const OUTPUT_DRAIN: Duration = Duration::from_secs(5);
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct WorkerProcessOptions {
|
|
pub python: String,
|
|
pub cwd: PathBuf,
|
|
pub args: Vec<String>,
|
|
/// Fixed port; 0 picks a free one once at start (kept across restarts).
|
|
pub port: u16,
|
|
/// Extra environment for the worker (over the app's own).
|
|
pub env: HashMap<String, String>,
|
|
pub restart_delay: Duration,
|
|
pub max_restart_delay: Duration,
|
|
/// Where output lines go; None: the app's log.
|
|
pub line_sink: Option<LineSink>,
|
|
}
|
|
|
|
impl WorkerProcessOptions {
|
|
pub fn new(python: impl Into<String>, cwd: impl Into<PathBuf>, args: Vec<String>) -> Self {
|
|
Self {
|
|
python: python.into(),
|
|
cwd: cwd.into(),
|
|
args,
|
|
port: 0,
|
|
env: HashMap::new(),
|
|
restart_delay: Duration::from_secs(1),
|
|
max_restart_delay: Duration::from_secs(30),
|
|
line_sink: None,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub enum ProcessEvent {
|
|
/// The worker printed READY: models are loaded and it accepts connections.
|
|
Ready,
|
|
Exited { code: Option<i32> },
|
|
}
|
|
|
|
pub fn free_port() -> std::io::Result<u16> {
|
|
Ok(std::net::TcpListener::bind("127.0.0.1:0")?.local_addr()?.port())
|
|
}
|
|
|
|
struct Shared {
|
|
stopping: bool,
|
|
child_pid: Option<u32>,
|
|
restarts: u32,
|
|
}
|
|
|
|
pub struct WorkerProcess {
|
|
opts: WorkerProcessOptions,
|
|
pub token: String,
|
|
port: u16,
|
|
shared: Arc<Mutex<Shared>>,
|
|
stop: Arc<Notify>,
|
|
events: broadcast::Sender<ProcessEvent>,
|
|
task: Mutex<Option<tokio::task::JoinHandle<()>>>,
|
|
}
|
|
|
|
impl WorkerProcess {
|
|
pub fn new(opts: WorkerProcessOptions) -> std::io::Result<Self> {
|
|
let port = if opts.port == 0 { free_port()? } else { opts.port };
|
|
let mut raw = [0u8; 32];
|
|
rand::fill(&mut raw);
|
|
let token = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(raw);
|
|
Ok(Self {
|
|
opts,
|
|
token,
|
|
port,
|
|
shared: Arc::new(Mutex::new(Shared { stopping: false, child_pid: None, restarts: 0 })),
|
|
stop: Arc::new(Notify::new()),
|
|
events: broadcast::channel(64).0,
|
|
task: Mutex::new(None),
|
|
})
|
|
}
|
|
|
|
pub fn url(&self) -> String {
|
|
format!("ws://127.0.0.1:{}", self.port)
|
|
}
|
|
|
|
pub fn port(&self) -> u16 {
|
|
self.port
|
|
}
|
|
|
|
pub fn restarts(&self) -> u32 {
|
|
self.shared.lock().unwrap().restarts
|
|
}
|
|
|
|
pub fn subscribe(&self) -> broadcast::Receiver<ProcessEvent> {
|
|
self.events.subscribe()
|
|
}
|
|
|
|
/// Spawn the worker and keep it running until `stop()`.
|
|
pub fn start(&self) {
|
|
let (opts, token, port) = (self.opts.clone(), self.token.clone(), self.port);
|
|
let (shared, stop, events) = (self.shared.clone(), self.stop.clone(), self.events.clone());
|
|
let sink = opts.line_sink.clone().unwrap_or_else(run_sink);
|
|
let task = tokio::spawn(async move {
|
|
let mut delay = opts.restart_delay;
|
|
loop {
|
|
let started = Instant::now();
|
|
let mut child = match spawn(&opts, &token, port) {
|
|
Ok(c) => c,
|
|
Err(e) => {
|
|
error_log!("worker", "voice worker could not start: {e}");
|
|
let _ = events.send(ProcessEvent::Exited { code: None });
|
|
if wait_or_stop(&stop, &shared, delay).await {
|
|
return;
|
|
}
|
|
delay = (delay * 2).min(opts.max_restart_delay);
|
|
continue;
|
|
}
|
|
};
|
|
shared.lock().unwrap().child_pid = child.id();
|
|
crate::info_log!("worker", "voice worker started pid={:?} port={port}", child.id());
|
|
let mut ready_rx = events.subscribe();
|
|
let mut readers = vec![];
|
|
// The worker's own logs hold timings and counts only (voice_worker.spec.md).
|
|
if let Some(out) = child.stdout.take() {
|
|
let (events, sink) = (events.clone(), sink.clone());
|
|
readers.push(tokio::spawn(async move {
|
|
let mut lines = BufReader::new(out).lines();
|
|
while let Ok(Some(line)) = lines.next_line().await {
|
|
if line.starts_with("READY ") {
|
|
crate::info_log!("worker", "voice worker READY after {} ms", started.elapsed().as_millis());
|
|
let _ = events.send(ProcessEvent::Ready);
|
|
} else {
|
|
(sink.0)(&line);
|
|
}
|
|
}
|
|
}));
|
|
}
|
|
if let Some(err) = child.stderr.take() {
|
|
let sink = sink.clone();
|
|
readers.push(tokio::spawn(async move {
|
|
let mut lines = BufReader::new(err).lines();
|
|
while let Ok(Some(line)) = lines.next_line().await {
|
|
(sink.0)(&line);
|
|
}
|
|
}));
|
|
}
|
|
let status = tokio::select! {
|
|
s = child.wait() => s.ok(),
|
|
_ = stop.notified() => {
|
|
let _ = child.kill().await;
|
|
let code = child.wait().await.ok().and_then(|s| s.code());
|
|
drain(readers).await;
|
|
let _ = events.send(ProcessEvent::Exited { code });
|
|
return;
|
|
}
|
|
};
|
|
// A worker that got as far as READY resets the backoff.
|
|
while let Ok(ev) = ready_rx.try_recv() {
|
|
if ev == ProcessEvent::Ready {
|
|
delay = opts.restart_delay;
|
|
}
|
|
}
|
|
let code = status.and_then(|s| s.code());
|
|
drain(readers).await;
|
|
shared.lock().unwrap().child_pid = None;
|
|
let _ = events.send(ProcessEvent::Exited { code });
|
|
if shared.lock().unwrap().stopping {
|
|
return;
|
|
}
|
|
error_log!("worker", "voice worker exited (code {code:?}); restarting in {} ms", delay.as_millis());
|
|
if wait_or_stop(&stop, &shared, delay).await {
|
|
return;
|
|
}
|
|
shared.lock().unwrap().restarts += 1;
|
|
delay = (delay * 2).min(opts.max_restart_delay);
|
|
}
|
|
});
|
|
*self.task.lock().unwrap() = Some(task);
|
|
}
|
|
|
|
/// Stop the worker and do not restart it. Returns once it has exited and its last output is logged.
|
|
pub async fn stop(&self) {
|
|
self.shared.lock().unwrap().stopping = true;
|
|
self.stop.notify_one();
|
|
let task = self.task.lock().unwrap().take();
|
|
if let Some(t) = task {
|
|
let _ = t.await;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Wait until the output readers have logged everything the exited worker wrote (at most
|
|
/// `OUTPUT_DRAIN`), so a stop returns only after its last lines were masked with this run's secrets
|
|
/// (`logs.rs`, `set_secrets_in_use`); readers still waiting after that are stopped.
|
|
async fn drain(readers: Vec<tokio::task::JoinHandle<()>>) {
|
|
let until = tokio::time::Instant::now() + OUTPUT_DRAIN;
|
|
for mut r in readers {
|
|
if tokio::time::timeout_at(until, &mut r).await.is_err() {
|
|
r.abort();
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Sleep `delay` unless stopped meanwhile (true = stopped).
|
|
async fn wait_or_stop(stop: &Notify, shared: &Mutex<Shared>, delay: Duration) -> bool {
|
|
if shared.lock().unwrap().stopping {
|
|
return true;
|
|
}
|
|
tokio::select! {
|
|
_ = tokio::time::sleep(delay) => shared.lock().unwrap().stopping,
|
|
_ = stop.notified() => true,
|
|
}
|
|
}
|
|
|
|
fn spawn(opts: &WorkerProcessOptions, token: &str, port: u16) -> std::io::Result<Child> {
|
|
let mut cmd = Command::new(&opts.python);
|
|
cmd.args(["-m", "voice_worker", "--host", "127.0.0.1", "--port", &port.to_string()])
|
|
.args(&opts.args)
|
|
.current_dir(&opts.cwd)
|
|
.envs(&opts.env)
|
|
.env("JARVIS_VOICE_TOKEN", token)
|
|
.env("PYTHONUNBUFFERED", "1")
|
|
.stdin(Stdio::null())
|
|
.stdout(Stdio::piped())
|
|
.stderr(Stdio::piped())
|
|
.kill_on_drop(true);
|
|
#[cfg(windows)]
|
|
{
|
|
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
|
|
cmd.creation_flags(CREATE_NO_WINDOW);
|
|
}
|
|
cmd.spawn()
|
|
}
|
|
|
|
/// The default sink of a worker started now: the app's log, masking also the values masked at
|
|
/// this moment (this run's secrets, `logs.rs` `set_secrets_in_use`), so a line it writes late (after
|
|
/// a stop gave up waiting for it, and the next start replaced the values in use) is still masked.
|
|
pub fn run_sink() -> LineSink {
|
|
let masks = crate::logs::installed().map(|h| h.masked_values()).unwrap_or_default();
|
|
LineSink(Arc::new(move |line: &str| worker_line(line, &masks)))
|
|
}
|
|
|
|
/// A line the voice worker printed: on stderr with debug logging, and always to the log hub (its
|
|
/// own category picks the tab; warnings and tracebacks reach the log screen without debug logging).
|
|
fn worker_line(line: &str, masks: &[String]) {
|
|
if crate::log::debug_enabled() {
|
|
eprintln!("{}", crate::log::line("worker", &crate::logs::redact(line, masks)));
|
|
}
|
|
if let Some(hub) = crate::logs::installed() {
|
|
hub.push_worker_masked(line, masks);
|
|
}
|
|
}
|