Stopping (and restarting) the voice worker now waits, at most 5 s, for its output readers to log what the exited worker wrote, so its last lines are masked with the secrets that run was started with before the next start replaces them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
153 lines
5.8 KiB
Rust
153 lines
5.8 KiB
Rust
//! The worker process with a stand-in `voice_worker` package run by a real Python, so the spawn line
|
|
//! is the one the app uses. JARVIS_TEST_PYTHON overrides the Python (default python3 / python).
|
|
|
|
use std::path::PathBuf;
|
|
use std::time::Duration;
|
|
|
|
use jarvis_app::worker_process::{ProcessEvent, WorkerProcess, WorkerProcessOptions};
|
|
use serde_json::Value;
|
|
|
|
const FAKE_MAIN: &str = r#"
|
|
import json, os, sys, time
|
|
out = os.environ["FAKE_OUT"]
|
|
runs = len(os.listdir(out))
|
|
with open(os.path.join(out, f"run{runs}.json"), "w") as f:
|
|
json.dump({"argv": sys.argv[1:], "token": os.environ.get("JARVIS_VOICE_TOKEN")}, f)
|
|
if os.environ.get("FAKE_CRASH_FIRST") == "1" and runs == 0:
|
|
sys.exit(3)
|
|
print("loading models...", flush=True)
|
|
print("READY 127.0.0.1:" + sys.argv[sys.argv.index("--port") + 1], flush=True)
|
|
if os.environ.get("FAKE_LAST_LINES") == "1":
|
|
for i in range(3):
|
|
print(f"line {i}", file=sys.stderr, flush=True)
|
|
print("LAST bad key old-api-key-1", file=sys.stderr, flush=True)
|
|
time.sleep(60)
|
|
"#;
|
|
|
|
fn python() -> String {
|
|
std::env::var("JARVIS_TEST_PYTHON").unwrap_or_else(|_| if cfg!(windows) { "python".into() } else { "python3".into() })
|
|
}
|
|
|
|
struct Fake {
|
|
root: PathBuf,
|
|
out: PathBuf,
|
|
}
|
|
|
|
impl Fake {
|
|
fn new(tag: &str) -> Self {
|
|
let root = std::env::temp_dir().join(format!("jarvis-fake-worker-{tag}-{}", std::process::id()));
|
|
let _ = std::fs::remove_dir_all(&root);
|
|
std::fs::create_dir_all(root.join("voice_worker")).unwrap();
|
|
std::fs::write(root.join("voice_worker/__init__.py"), "").unwrap();
|
|
std::fs::write(root.join("voice_worker/__main__.py"), FAKE_MAIN).unwrap();
|
|
let out = root.join("out");
|
|
std::fs::create_dir_all(&out).unwrap();
|
|
Self { root, out }
|
|
}
|
|
fn run(&self, n: usize) -> Value {
|
|
serde_json::from_str(&std::fs::read_to_string(self.out.join(format!("run{n}.json"))).unwrap()).unwrap()
|
|
}
|
|
fn opts(&self, crash_first: bool) -> WorkerProcessOptions {
|
|
let mut o = WorkerProcessOptions::new(python(), &self.root, vec!["--melo-python".into(), "/x/python".into()]);
|
|
o.env.insert("FAKE_OUT".into(), self.out.display().to_string());
|
|
if crash_first {
|
|
o.env.insert("FAKE_CRASH_FIRST".into(), "1".into());
|
|
}
|
|
o.restart_delay = Duration::from_millis(50);
|
|
o
|
|
}
|
|
}
|
|
|
|
impl Drop for Fake {
|
|
fn drop(&mut self) {
|
|
let _ = std::fs::remove_dir_all(&self.root);
|
|
}
|
|
}
|
|
|
|
async fn wait_for(rx: &mut tokio::sync::broadcast::Receiver<ProcessEvent>, f: impl Fn(&ProcessEvent) -> bool) -> ProcessEvent {
|
|
tokio::time::timeout(Duration::from_secs(10), async {
|
|
loop {
|
|
let e = rx.recv().await.unwrap();
|
|
if f(&e) {
|
|
return e;
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.expect("timed out")
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn runs_the_worker_on_loopback_with_the_token_only_in_the_environment() {
|
|
let fake = Fake::new("env");
|
|
let p = WorkerProcess::new(fake.opts(false)).unwrap();
|
|
let mut rx = p.subscribe();
|
|
p.start();
|
|
wait_for(&mut rx, |e| *e == ProcessEvent::Ready).await;
|
|
let run = fake.run(0);
|
|
let port = p.port().to_string();
|
|
let argv: Vec<String> = serde_json::from_value(run["argv"].clone()).unwrap();
|
|
assert_eq!(argv, ["--host", "127.0.0.1", "--port", &port, "--melo-python", "/x/python"]);
|
|
assert_eq!(run["token"], p.token.as_str());
|
|
assert!(p.token.len() >= 40);
|
|
assert!(!argv.join(" ").contains(&p.token));
|
|
assert!(p.url().starts_with("ws://127.0.0.1:"));
|
|
p.stop().await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_dead_worker_restarts_on_the_same_port_with_the_same_token() {
|
|
let fake = Fake::new("restart");
|
|
let p = WorkerProcess::new(fake.opts(true)).unwrap();
|
|
let mut rx = p.subscribe();
|
|
p.start();
|
|
assert_eq!(wait_for(&mut rx, |e| matches!(e, ProcessEvent::Exited { .. })).await, ProcessEvent::Exited { code: Some(3) });
|
|
wait_for(&mut rx, |e| *e == ProcessEvent::Ready).await;
|
|
assert_eq!(p.restarts(), 1);
|
|
assert_eq!(fake.run(1)["argv"], fake.run(0)["argv"]);
|
|
assert_eq!(fake.run(1)["token"], fake.run(0)["token"]);
|
|
p.stop().await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn stop_ends_the_worker_for_good() {
|
|
let fake = Fake::new("stop");
|
|
let p = WorkerProcess::new(fake.opts(false)).unwrap();
|
|
let mut rx = p.subscribe();
|
|
p.start();
|
|
wait_for(&mut rx, |e| *e == ProcessEvent::Ready).await;
|
|
p.stop().await;
|
|
assert!(matches!(rx.recv().await.unwrap(), ProcessEvent::Exited { .. }));
|
|
tokio::time::sleep(Duration::from_millis(200)).await;
|
|
assert!(rx.try_recv().is_err(), "no restart after stop");
|
|
assert_eq!(p.restarts(), 0);
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn stop_returns_only_after_the_last_output_is_logged() {
|
|
// A slow log (each line takes 300 ms) and a worker whose last lines are still unread when it is
|
|
// stopped: they must be handled before stop returns, so the next start's secrets (logs.rs,
|
|
// set_secrets_in_use) never apply to them.
|
|
let fake = Fake::new("drain");
|
|
let mut o = fake.opts(false);
|
|
o.env.insert("FAKE_LAST_LINES".into(), "1".into());
|
|
let seen = std::sync::Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
|
|
let s2 = seen.clone();
|
|
o.line_sink = Some(jarvis_app::worker_process::LineSink(std::sync::Arc::new(move |l: &str| {
|
|
std::thread::sleep(Duration::from_millis(300));
|
|
s2.lock().unwrap().push(l.to_owned());
|
|
})));
|
|
let p = WorkerProcess::new(o).unwrap();
|
|
let mut rx = p.subscribe();
|
|
p.start();
|
|
wait_for(&mut rx, |e| *e == ProcessEvent::Ready).await;
|
|
tokio::time::timeout(Duration::from_secs(10), async {
|
|
while seen.lock().unwrap().is_empty() {
|
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
|
}
|
|
}).await.expect("no output");
|
|
p.stop().await;
|
|
let lines = seen.lock().unwrap().clone();
|
|
assert!(lines.iter().any(|l| l.contains("LAST")), "{lines:?}");
|
|
}
|