fix(app): a worker stop returns only after its last output is logged

Stopping (and restarting) the voice worker now waits, at most 5 s, for its output readers to log
what the exited worker wrote, so its last lines are masked with the secrets that run was started
with before the next start replaces them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-13 10:15:20 +09:00
parent 648e8304b0
commit fba258821b
3 changed files with 77 additions and 10 deletions

View File

@@ -16,6 +16,20 @@ use tokio::sync::{broadcast, Notify};
use crate::error_log;
/// Where the worker's output lines go (default: the app's log, `worker_line`).
#[derive(Clone)]
pub struct LineSink(pub Arc<dyn Fn(&str) + Send + Sync>);
impl std::fmt::Debug for LineSink {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("LineSink")
}
}
/// How long a stop or restart waits for the worker's last output to be logged. A child of the worker
/// (an MCP server) may hold its output open after it is gone; then the rest is not read.
const OUTPUT_DRAIN: Duration = Duration::from_secs(5);
#[derive(Debug, Clone)]
pub struct WorkerProcessOptions {
pub python: String,
@@ -27,6 +41,8 @@ pub struct WorkerProcessOptions {
pub env: HashMap<String, String>,
pub restart_delay: Duration,
pub max_restart_delay: Duration,
/// Where output lines go; None: the app's log.
pub line_sink: Option<LineSink>,
}
impl WorkerProcessOptions {
@@ -39,6 +55,7 @@ impl WorkerProcessOptions {
env: HashMap::new(),
restart_delay: Duration::from_secs(1),
max_restart_delay: Duration::from_secs(30),
line_sink: None,
}
}
}
@@ -126,34 +143,37 @@ impl WorkerProcess {
shared.lock().unwrap().child_pid = child.id();
crate::info_log!("worker", "voice worker started pid={:?} port={port}", child.id());
let mut ready_rx = events.subscribe();
let sink = opts.line_sink.clone().unwrap_or_else(|| LineSink(Arc::new(worker_line)));
let mut readers = vec![];
// The worker's own logs hold timings and counts only (voice_worker.spec.md).
if let Some(out) = child.stdout.take() {
let events = events.clone();
tokio::spawn(async move {
let (events, sink) = (events.clone(), sink.clone());
readers.push(tokio::spawn(async move {
let mut lines = BufReader::new(out).lines();
while let Ok(Some(line)) = lines.next_line().await {
if line.starts_with("READY ") {
crate::info_log!("worker", "voice worker READY after {} ms", started.elapsed().as_millis());
let _ = events.send(ProcessEvent::Ready);
} else {
worker_line(&line);
(sink.0)(&line);
}
}
});
}));
}
if let Some(err) = child.stderr.take() {
tokio::spawn(async move {
readers.push(tokio::spawn(async move {
let mut lines = BufReader::new(err).lines();
while let Ok(Some(line)) = lines.next_line().await {
worker_line(&line);
(sink.0)(&line);
}
});
}));
}
let status = tokio::select! {
s = child.wait() => s.ok(),
_ = stop.notified() => {
let _ = child.kill().await;
let code = child.wait().await.ok().and_then(|s| s.code());
drain(readers).await;
let _ = events.send(ProcessEvent::Exited { code });
return;
}
@@ -165,6 +185,7 @@ impl WorkerProcess {
}
}
let code = status.and_then(|s| s.code());
drain(readers).await;
shared.lock().unwrap().child_pid = None;
let _ = events.send(ProcessEvent::Exited { code });
if shared.lock().unwrap().stopping {
@@ -181,7 +202,7 @@ impl WorkerProcess {
*self.task.lock().unwrap() = Some(task);
}
/// Stop the worker and do not restart it. Returns once it has exited.
/// Stop the worker and do not restart it. Returns once it has exited and its last output is logged.
pub async fn stop(&self) {
self.shared.lock().unwrap().stopping = true;
self.stop.notify_one();
@@ -192,6 +213,18 @@ impl WorkerProcess {
}
}
/// Wait until the output readers have logged everything the exited worker wrote (at most
/// `OUTPUT_DRAIN`), so a stop returns only after its last lines were masked with this run's secrets
/// (`logs.rs`, `set_secrets_in_use`); readers still waiting after that are stopped.
async fn drain(readers: Vec<tokio::task::JoinHandle<()>>) {
let until = tokio::time::Instant::now() + OUTPUT_DRAIN;
for mut r in readers {
if tokio::time::timeout_at(until, &mut r).await.is_err() {
r.abort();
}
}
}
/// Sleep `delay` unless stopped meanwhile (true = stopped).
async fn wait_or_stop(stop: &Notify, shared: &Mutex<Shared>, delay: Duration) -> bool {
if shared.lock().unwrap().stopping {