fix(app): a worker stop returns only after its last output is logged
Stopping (and restarting) the voice worker now waits, at most 5 s, for its output readers to log what the exited worker wrote, so its last lines are masked with the secrets that run was started with before the next start replaces them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -86,7 +86,9 @@ The app starts `python -m voice_worker --host 127.0.0.1 --port <port> <args>` in
|
||||
`JARVIS_VOICE_TOKEN`. The port is a free loopback port chosen once at start. If the worker exits, it is
|
||||
started again on the same port with the same token after 1 s, doubling to at most 30 s while it keeps
|
||||
failing (reset once a run printed `READY`). Stopping kills it; MeloTTS's child process ends when its
|
||||
input closes.
|
||||
input closes. A stop (and a restart) first logs everything the exited worker wrote, waiting at most
|
||||
5 s (a child of the worker may hold its output open): a stop returns only after its last lines were
|
||||
masked with the secrets that run was started with (`admin.spec.md`, "Logs").
|
||||
|
||||
## Worker link
|
||||
|
||||
@@ -225,7 +227,7 @@ registered until it is removed (M5 setup).
|
||||
silent at both ends, below its peak), speaker limit, chat access and splitting, settings, frames; the worker
|
||||
link against a fake server (hello, events, reconnect, failed TTS); the worker process with a
|
||||
stand-in `voice_worker` package run by a real Python (token only in the environment, restart on the
|
||||
same port, stop).
|
||||
same port, stop, the last output logged before a stop returns).
|
||||
- `JARVIS_VOICE_PYTHON=<venv python> cargo test --test worker_protocol` runs the worker link and
|
||||
playback against the real Python worker server with fake engines (`tests/fixtures/fake_engine_worker.py`).
|
||||
- `tests/live_discord.rs` (opt-in, `JARVIS_LIVE_DISCORD=1`, see the file header): two bots in one
|
||||
|
||||
@@ -16,6 +16,20 @@ use tokio::sync::{broadcast, Notify};
|
||||
|
||||
use crate::error_log;
|
||||
|
||||
/// Where the worker's output lines go (default: the app's log, `worker_line`).
|
||||
#[derive(Clone)]
|
||||
pub struct LineSink(pub Arc<dyn Fn(&str) + Send + Sync>);
|
||||
|
||||
impl std::fmt::Debug for LineSink {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("LineSink")
|
||||
}
|
||||
}
|
||||
|
||||
/// How long a stop or restart waits for the worker's last output to be logged. A child of the worker
|
||||
/// (an MCP server) may hold its output open after it is gone; then the rest is not read.
|
||||
const OUTPUT_DRAIN: Duration = Duration::from_secs(5);
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct WorkerProcessOptions {
|
||||
pub python: String,
|
||||
@@ -27,6 +41,8 @@ pub struct WorkerProcessOptions {
|
||||
pub env: HashMap<String, String>,
|
||||
pub restart_delay: Duration,
|
||||
pub max_restart_delay: Duration,
|
||||
/// Where output lines go; None: the app's log.
|
||||
pub line_sink: Option<LineSink>,
|
||||
}
|
||||
|
||||
impl WorkerProcessOptions {
|
||||
@@ -39,6 +55,7 @@ impl WorkerProcessOptions {
|
||||
env: HashMap::new(),
|
||||
restart_delay: Duration::from_secs(1),
|
||||
max_restart_delay: Duration::from_secs(30),
|
||||
line_sink: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -126,34 +143,37 @@ impl WorkerProcess {
|
||||
shared.lock().unwrap().child_pid = child.id();
|
||||
crate::info_log!("worker", "voice worker started pid={:?} port={port}", child.id());
|
||||
let mut ready_rx = events.subscribe();
|
||||
let sink = opts.line_sink.clone().unwrap_or_else(|| LineSink(Arc::new(worker_line)));
|
||||
let mut readers = vec![];
|
||||
// The worker's own logs hold timings and counts only (voice_worker.spec.md).
|
||||
if let Some(out) = child.stdout.take() {
|
||||
let events = events.clone();
|
||||
tokio::spawn(async move {
|
||||
let (events, sink) = (events.clone(), sink.clone());
|
||||
readers.push(tokio::spawn(async move {
|
||||
let mut lines = BufReader::new(out).lines();
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
if line.starts_with("READY ") {
|
||||
crate::info_log!("worker", "voice worker READY after {} ms", started.elapsed().as_millis());
|
||||
let _ = events.send(ProcessEvent::Ready);
|
||||
} else {
|
||||
worker_line(&line);
|
||||
(sink.0)(&line);
|
||||
}
|
||||
}
|
||||
});
|
||||
}));
|
||||
}
|
||||
if let Some(err) = child.stderr.take() {
|
||||
tokio::spawn(async move {
|
||||
readers.push(tokio::spawn(async move {
|
||||
let mut lines = BufReader::new(err).lines();
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
worker_line(&line);
|
||||
(sink.0)(&line);
|
||||
}
|
||||
});
|
||||
}));
|
||||
}
|
||||
let status = tokio::select! {
|
||||
s = child.wait() => s.ok(),
|
||||
_ = stop.notified() => {
|
||||
let _ = child.kill().await;
|
||||
let code = child.wait().await.ok().and_then(|s| s.code());
|
||||
drain(readers).await;
|
||||
let _ = events.send(ProcessEvent::Exited { code });
|
||||
return;
|
||||
}
|
||||
@@ -165,6 +185,7 @@ impl WorkerProcess {
|
||||
}
|
||||
}
|
||||
let code = status.and_then(|s| s.code());
|
||||
drain(readers).await;
|
||||
shared.lock().unwrap().child_pid = None;
|
||||
let _ = events.send(ProcessEvent::Exited { code });
|
||||
if shared.lock().unwrap().stopping {
|
||||
@@ -181,7 +202,7 @@ impl WorkerProcess {
|
||||
*self.task.lock().unwrap() = Some(task);
|
||||
}
|
||||
|
||||
/// Stop the worker and do not restart it. Returns once it has exited.
|
||||
/// Stop the worker and do not restart it. Returns once it has exited and its last output is logged.
|
||||
pub async fn stop(&self) {
|
||||
self.shared.lock().unwrap().stopping = true;
|
||||
self.stop.notify_one();
|
||||
@@ -192,6 +213,18 @@ impl WorkerProcess {
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait until the output readers have logged everything the exited worker wrote (at most
|
||||
/// `OUTPUT_DRAIN`), so a stop returns only after its last lines were masked with this run's secrets
|
||||
/// (`logs.rs`, `set_secrets_in_use`); readers still waiting after that are stopped.
|
||||
async fn drain(readers: Vec<tokio::task::JoinHandle<()>>) {
|
||||
let until = tokio::time::Instant::now() + OUTPUT_DRAIN;
|
||||
for mut r in readers {
|
||||
if tokio::time::timeout_at(until, &mut r).await.is_err() {
|
||||
r.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Sleep `delay` unless stopped meanwhile (true = stopped).
|
||||
async fn wait_or_stop(stop: &Notify, shared: &Mutex<Shared>, delay: Duration) -> bool {
|
||||
if shared.lock().unwrap().stopping {
|
||||
|
||||
@@ -17,6 +17,10 @@ if os.environ.get("FAKE_CRASH_FIRST") == "1" and runs == 0:
|
||||
sys.exit(3)
|
||||
print("loading models...", flush=True)
|
||||
print("READY 127.0.0.1:" + sys.argv[sys.argv.index("--port") + 1], flush=True)
|
||||
if os.environ.get("FAKE_LAST_LINES") == "1":
|
||||
for i in range(3):
|
||||
print(f"line {i}", file=sys.stderr, flush=True)
|
||||
print("LAST bad key old-api-key-1", file=sys.stderr, flush=True)
|
||||
time.sleep(60)
|
||||
"#;
|
||||
|
||||
@@ -118,3 +122,31 @@ async fn stop_ends_the_worker_for_good() {
|
||||
assert!(rx.try_recv().is_err(), "no restart after stop");
|
||||
assert_eq!(p.restarts(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn stop_returns_only_after_the_last_output_is_logged() {
|
||||
// A slow log (each line takes 300 ms) and a worker whose last lines are still unread when it is
|
||||
// stopped: they must be handled before stop returns, so the next start's secrets (logs.rs,
|
||||
// set_secrets_in_use) never apply to them.
|
||||
let fake = Fake::new("drain");
|
||||
let mut o = fake.opts(false);
|
||||
o.env.insert("FAKE_LAST_LINES".into(), "1".into());
|
||||
let seen = std::sync::Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
|
||||
let s2 = seen.clone();
|
||||
o.line_sink = Some(jarvis_app::worker_process::LineSink(std::sync::Arc::new(move |l: &str| {
|
||||
std::thread::sleep(Duration::from_millis(300));
|
||||
s2.lock().unwrap().push(l.to_owned());
|
||||
})));
|
||||
let p = WorkerProcess::new(o).unwrap();
|
||||
let mut rx = p.subscribe();
|
||||
p.start();
|
||||
wait_for(&mut rx, |e| *e == ProcessEvent::Ready).await;
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while seen.lock().unwrap().is_empty() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
}).await.expect("no output");
|
||||
p.stop().await;
|
||||
let lines = seen.lock().unwrap().clone();
|
||||
assert!(lines.iter().any(|l| l.contains("LAST")), "{lines:?}");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user