Files
javis_bot/app/src/worker_process.rs
javis-bot 71f9b0cd54 fix(app): a worker run's late lines keep its run's secrets masked
Each worker run's log sink keeps the values masked when it started (its secrets in use), so a
line still being logged after a stop gave up waiting, and after the next start replaced the values
in use, is still masked. The debug echo on stderr is masked the same way.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 10:19:15 +09:00

279 lines
10 KiB
Rust

//! Starts the Python voice worker as a child process and restarts it if it dies.
//!
//! Each app launch makes a new random token and hands it over in the environment
//! (`JARVIS_VOICE_TOKEN`), never on the command line. The worker listens on 127.0.0.1 only.
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Stdio;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use base64::Engine;
use tokio::io::{AsyncBufReadExt, BufReader};
use tokio::process::{Child, Command};
use tokio::sync::{broadcast, Notify};
use crate::error_log;
/// Where the worker's output lines go (default: the app's log, `worker_line`).
#[derive(Clone)]
pub struct LineSink(pub Arc<dyn Fn(&str) + Send + Sync>);
impl std::fmt::Debug for LineSink {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("LineSink")
}
}
/// How long a stop or restart waits for the worker's last output to be logged. A child of the worker
/// (an MCP server) may hold its output open after it is gone; then the rest is not read. A line
/// still being logged after that is masked with its run's values all the same (`run_sink`).
const OUTPUT_DRAIN: Duration = Duration::from_secs(5);
#[derive(Debug, Clone)]
pub struct WorkerProcessOptions {
pub python: String,
pub cwd: PathBuf,
pub args: Vec<String>,
/// Fixed port; 0 picks a free one once at start (kept across restarts).
pub port: u16,
/// Extra environment for the worker (over the app's own).
pub env: HashMap<String, String>,
pub restart_delay: Duration,
pub max_restart_delay: Duration,
/// Where output lines go; None: the app's log.
pub line_sink: Option<LineSink>,
}
impl WorkerProcessOptions {
pub fn new(python: impl Into<String>, cwd: impl Into<PathBuf>, args: Vec<String>) -> Self {
Self {
python: python.into(),
cwd: cwd.into(),
args,
port: 0,
env: HashMap::new(),
restart_delay: Duration::from_secs(1),
max_restart_delay: Duration::from_secs(30),
line_sink: None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ProcessEvent {
/// The worker printed READY: models are loaded and it accepts connections.
Ready,
Exited { code: Option<i32> },
}
pub fn free_port() -> std::io::Result<u16> {
Ok(std::net::TcpListener::bind("127.0.0.1:0")?.local_addr()?.port())
}
struct Shared {
stopping: bool,
child_pid: Option<u32>,
restarts: u32,
}
pub struct WorkerProcess {
opts: WorkerProcessOptions,
pub token: String,
port: u16,
shared: Arc<Mutex<Shared>>,
stop: Arc<Notify>,
events: broadcast::Sender<ProcessEvent>,
task: Mutex<Option<tokio::task::JoinHandle<()>>>,
}
impl WorkerProcess {
pub fn new(opts: WorkerProcessOptions) -> std::io::Result<Self> {
let port = if opts.port == 0 { free_port()? } else { opts.port };
let mut raw = [0u8; 32];
rand::fill(&mut raw);
let token = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(raw);
Ok(Self {
opts,
token,
port,
shared: Arc::new(Mutex::new(Shared { stopping: false, child_pid: None, restarts: 0 })),
stop: Arc::new(Notify::new()),
events: broadcast::channel(64).0,
task: Mutex::new(None),
})
}
pub fn url(&self) -> String {
format!("ws://127.0.0.1:{}", self.port)
}
pub fn port(&self) -> u16 {
self.port
}
pub fn restarts(&self) -> u32 {
self.shared.lock().unwrap().restarts
}
pub fn subscribe(&self) -> broadcast::Receiver<ProcessEvent> {
self.events.subscribe()
}
/// Spawn the worker and keep it running until `stop()`.
pub fn start(&self) {
let (opts, token, port) = (self.opts.clone(), self.token.clone(), self.port);
let (shared, stop, events) = (self.shared.clone(), self.stop.clone(), self.events.clone());
let sink = opts.line_sink.clone().unwrap_or_else(run_sink);
let task = tokio::spawn(async move {
let mut delay = opts.restart_delay;
loop {
let started = Instant::now();
let mut child = match spawn(&opts, &token, port) {
Ok(c) => c,
Err(e) => {
error_log!("worker", "voice worker could not start: {e}");
let _ = events.send(ProcessEvent::Exited { code: None });
if wait_or_stop(&stop, &shared, delay).await {
return;
}
delay = (delay * 2).min(opts.max_restart_delay);
continue;
}
};
shared.lock().unwrap().child_pid = child.id();
crate::info_log!("worker", "voice worker started pid={:?} port={port}", child.id());
let mut ready_rx = events.subscribe();
let mut readers = vec![];
// The worker's own logs hold timings and counts only (voice_worker.spec.md).
if let Some(out) = child.stdout.take() {
let (events, sink) = (events.clone(), sink.clone());
readers.push(tokio::spawn(async move {
let mut lines = BufReader::new(out).lines();
while let Ok(Some(line)) = lines.next_line().await {
if line.starts_with("READY ") {
crate::info_log!("worker", "voice worker READY after {} ms", started.elapsed().as_millis());
let _ = events.send(ProcessEvent::Ready);
} else {
(sink.0)(&line);
}
}
}));
}
if let Some(err) = child.stderr.take() {
let sink = sink.clone();
readers.push(tokio::spawn(async move {
let mut lines = BufReader::new(err).lines();
while let Ok(Some(line)) = lines.next_line().await {
(sink.0)(&line);
}
}));
}
let status = tokio::select! {
s = child.wait() => s.ok(),
_ = stop.notified() => {
let _ = child.kill().await;
let code = child.wait().await.ok().and_then(|s| s.code());
drain(readers).await;
let _ = events.send(ProcessEvent::Exited { code });
return;
}
};
// A worker that got as far as READY resets the backoff.
while let Ok(ev) = ready_rx.try_recv() {
if ev == ProcessEvent::Ready {
delay = opts.restart_delay;
}
}
let code = status.and_then(|s| s.code());
drain(readers).await;
shared.lock().unwrap().child_pid = None;
let _ = events.send(ProcessEvent::Exited { code });
if shared.lock().unwrap().stopping {
return;
}
error_log!("worker", "voice worker exited (code {code:?}); restarting in {} ms", delay.as_millis());
if wait_or_stop(&stop, &shared, delay).await {
return;
}
shared.lock().unwrap().restarts += 1;
delay = (delay * 2).min(opts.max_restart_delay);
}
});
*self.task.lock().unwrap() = Some(task);
}
/// Stop the worker and do not restart it. Returns once it has exited and its last output is logged.
pub async fn stop(&self) {
self.shared.lock().unwrap().stopping = true;
self.stop.notify_one();
let task = self.task.lock().unwrap().take();
if let Some(t) = task {
let _ = t.await;
}
}
}
/// Wait until the output readers have logged everything the exited worker wrote (at most
/// `OUTPUT_DRAIN`), so a stop returns only after its last lines were masked with this run's secrets
/// (`logs.rs`, `set_secrets_in_use`); readers still waiting after that are stopped.
async fn drain(readers: Vec<tokio::task::JoinHandle<()>>) {
let until = tokio::time::Instant::now() + OUTPUT_DRAIN;
for mut r in readers {
if tokio::time::timeout_at(until, &mut r).await.is_err() {
r.abort();
}
}
}
/// Sleep `delay` unless stopped meanwhile (true = stopped).
async fn wait_or_stop(stop: &Notify, shared: &Mutex<Shared>, delay: Duration) -> bool {
if shared.lock().unwrap().stopping {
return true;
}
tokio::select! {
_ = tokio::time::sleep(delay) => shared.lock().unwrap().stopping,
_ = stop.notified() => true,
}
}
fn spawn(opts: &WorkerProcessOptions, token: &str, port: u16) -> std::io::Result<Child> {
let mut cmd = Command::new(&opts.python);
cmd.args(["-m", "voice_worker", "--host", "127.0.0.1", "--port", &port.to_string()])
.args(&opts.args)
.current_dir(&opts.cwd)
.envs(&opts.env)
.env("JARVIS_VOICE_TOKEN", token)
.env("PYTHONUNBUFFERED", "1")
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true);
#[cfg(windows)]
{
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
cmd.creation_flags(CREATE_NO_WINDOW);
}
cmd.spawn()
}
/// The default sink of a worker started now: the app's log, masking also the values masked at
/// this moment (this run's secrets, `logs.rs` `set_secrets_in_use`), so a line it writes late (after
/// a stop gave up waiting for it, and the next start replaced the values in use) is still masked.
pub fn run_sink() -> LineSink {
let masks = crate::logs::installed().map(|h| h.masked_values()).unwrap_or_default();
LineSink(Arc::new(move |line: &str| worker_line(line, &masks)))
}
/// A line the voice worker printed: on stderr with debug logging, and always to the log hub (its
/// own category picks the tab; warnings and tracebacks reach the log screen without debug logging).
fn worker_line(line: &str, masks: &[String]) {
if crate::log::debug_enabled() {
eprintln!("{}", crate::log::line("worker", &crate::logs::redact(line, masks)));
}
if let Some(hub) = crate::logs::installed() {
hub.push_worker_masked(line, masks);
}
}